def trained_model(): args = [ "--num_users", str(100), "--num_items", str(100), "--max_epochs", str(100), "--num_workers", str(4), ] if th.cuda.is_available(): args = args + ["--gpus", str(1)] parser = ArgumentParser(add_help=False) parser = Trainer.add_argparse_args(parser) parser = ImplicitMatrixFactorization.add_model_specific_args(parser) parser = TestDataModule.add_dataset_specific_args(parser) parsed_args = parser.parse_args(args) model = ImplicitMatrixFactorization(parsed_args) test_dataset = TestDataModule(parsed_args.batch_size, parsed_args.num_workers) trainer = Trainer.from_argparse_args(parsed_args) trainer.fit(model, test_dataset) return model
def test_precision_parsed_correctly(arg, expected): """Test to ensure that the precision flag is passed correctly when adding argparse args.""" parser = ArgumentParser() parser = Trainer.add_argparse_args(parser) fake_argv = [arg] args = parser.parse_args(fake_argv) assert args.precision == expected
def main(): seed_everything(4321) parser = ArgumentParser(add_help=False) parser = Trainer.add_argparse_args(parser) parser.add_argument("--trainer_method", default="fit") parser.add_argument("--tmpdir") parser.add_argument("--workdir") parser.set_defaults(gpus=2) parser.set_defaults(strategy="ddp") args = parser.parse_args() dm = ClassifDataModule() model = ClassificationModel() trainer = Trainer.from_argparse_args(args) if args.trainer_method == "fit": trainer.fit(model, datamodule=dm) result = None elif args.trainer_method == "test": result = trainer.test(model, datamodule=dm) elif args.trainer_method == "fit_test": trainer.fit(model, datamodule=dm) result = trainer.test(model, datamodule=dm) else: raise ValueError(f"Unsupported: {args.trainer_method}") result_ext = { "status": "complete", "method": args.trainer_method, "result": result } file_path = os.path.join(args.tmpdir, "ddp.result") torch.save(result_ext, file_path)
def run_fit(): import pprint parser = ArgumentParser() parser = Trainer.add_argparse_args(parser) parser = WellnessClassifier.add_model_specific_args(parser) args = parser.parse_args() args.gpus = 2 args.max_epochs = 25 args.train_batch_size = 8 args.batch_size = 8 pprint.pprint(vars(args)) wellness_dm = WellnessDataModule(args) wellness_dm.prepare_data() print("num labels=", wellness_dm.num_labels) model = WellnessClassifier(args, wellness_dm.num_labels) wellness_dm = WellnessDataModule(args) wellness_dm.prepare_data() trainer = Trainer.from_argparse_args( args, early_stop_callback=early_stop_callback) trainer.fit(model, wellness_dm)
def main(): seed_everything(1234) parser = ArgumentParser(add_help=False) parser = Trainer.add_argparse_args(parser) parser.add_argument('--trainer_method', default='fit') parser.add_argument('--tmpdir') parser.add_argument('--workdir') parser.set_defaults(gpus=2) parser.set_defaults(accelerator="ddp") args = parser.parse_args() model = EvalModelTemplate() trainer = Trainer.from_argparse_args(args) result = {} if args.trainer_method == 'fit': trainer.fit(model) result = {'status': 'complete', 'method': args.trainer_method, 'result': None} if args.trainer_method == 'test': result = trainer.test(model) result = {'status': 'complete', 'method': args.trainer_method, 'result': result} if args.trainer_method == 'fit_test': trainer.fit(model) result = trainer.test(model) result = {'status': 'complete', 'method': args.trainer_method, 'result': result} if len(result) > 0: file_path = os.path.join(args.tmpdir, 'ddp.result') torch.save(result, file_path)
def main(): parser = get_program_level_args() parser = LitFood101.add_model_specific_args(parser) parser = Trainer.add_argparse_args(parser) args = parser.parse_args() seed_everything(args.seed) checkpoint_callback = ModelCheckpoint(monitor="avg_val_acc", mode="max") trainer = Trainer.from_argparse_args( args, deterministic=True, benchmark=False, checkpoint_callback=checkpoint_callback, precision=16 if args.amp_level != "O0" else 32, ) # create model model = resnet18(pretrained=True) if args.use_knowledge_distillation: teacher_model = resnet50(pretrained=False) model = LitFood101KD(model, teacher_model, args) else: model = LitFood101(model, args) if args.evaluate: checkpoint = torch.load(args.checkpoint) model.load_state_dict(checkpoint["state_dict"]) trainer.test(model, test_dataloaders=model.test_dataloader()) return 0 trainer.fit(model) trainer.test()
def get_args(): parser = argparse.ArgumentParser() parser.add_argument('-s', '--sweep', action='store_true', help='Run a hyperparameter sweep over all options') # DataModule args parser = MNISTDataModule.add_argparse_args(parser) # Trainer args (https://pytorch-lightning.readthedocs.io/en/latest/common/trainer.html#trainer-flags) parser = Trainer.add_argparse_args(parser) # Set some sane defaults for x in parser._actions: if x.dest == 'gpus': x.default = 1 if x.dest == 'max_epochs': x.default = 100 # TestTube args - hyperparam parser & slurm info parser = HyperOptArgumentParser(strategy='grid_search', add_help=False, parents=[parser]) parser.add_argument('--test_tube_exp_name', default='sweep_test') parser.add_argument('--log_path', default='./pytorch-slurm') # LightningModule args (hyperparameters) parser = MNISTClassifier.add_model_specific_args(parser) args = parser.parse_args() return args
def main(): parser = ArgumentParser() # add task level args parser = add_common_specific_args(parser) parser = add_tune_specific_args(parser) parser = add_task_specific_args(parser) # add model specific args parser = Model.add_model_specific_args(parser) parser = optimization.add_optimizer_specific_args(parser) parser = Trainer.add_argparse_args(parser) # set task specific args parser.set_defaults(gradient_clip_val=1.0, min_epochs=1, max_epochs=10) parser.set_defaults(num_labels=2) args = parser.parse_args() if args.build_dataset: build_distill_dataset(args) elif args.tune: tune_train(args, model_class=Model, task_info=task_info) else: common_train(args, model_class=Model, task_info=task_info)
def cli_main() -> None: from pl_bolts.datamodules.sklearn_datamodule import SklearnDataModule from pl_bolts.utils import _SKLEARN_AVAILABLE seed_everything(1234) # create dataset if _SKLEARN_AVAILABLE: from sklearn.datasets import load_diabetes else: # pragma: no cover raise ModuleNotFoundError( 'You want to use `sklearn` which is not installed yet, install it with `pip install sklearn`.' ) # args parser = ArgumentParser() parser = LinearRegression.add_model_specific_args(parser) parser = Trainer.add_argparse_args(parser) args = parser.parse_args() # model model = LinearRegression(input_dim=10, l1_strength=1, l2_strength=1) # model = LinearRegression(**vars(args)) # data X, y = load_diabetes(return_X_y=True) # these are numpy arrays loaders = SklearnDataModule(X, y, batch_size=args.batch_size) # train trainer = Trainer.from_argparse_args(args) trainer.fit(model, train_dataloader=loaders.train_dataloader(), val_dataloaders=loaders.val_dataloader())
def cli_main(args=None): from pl_bolts.datamodules import CIFAR10DataModule, ImagenetDataModule, STL10DataModule seed_everything() parser = ArgumentParser() parser.add_argument("--dataset", default="cifar10", type=str, choices=["cifar10", "stl10", "imagenet"]) script_args, _ = parser.parse_known_args(args) if script_args.dataset == "cifar10": dm_cls = CIFAR10DataModule elif script_args.dataset == "stl10": dm_cls = STL10DataModule elif script_args.dataset == "imagenet": dm_cls = ImagenetDataModule else: raise ValueError(f"undefined dataset {script_args.dataset}") parser = VAE.add_model_specific_args(parser) parser = Trainer.add_argparse_args(parser) args = parser.parse_args(args) dm = dm_cls.from_argparse_args(args) args.input_height = dm.size()[-1] if args.max_steps == -1: args.max_steps = None model = VAE(**vars(args)) trainer = Trainer.from_argparse_args(args) trainer.fit(model, datamodule=dm) return dm, model, trainer
def main(): """main""" parser = get_parser() parser = Trainer.add_argparse_args(parser) args = parser.parse_args() model = BertClassificationTask(args) checkpoint_callback = ModelCheckpoint( filepath=os.path.join(args.save_path, 'checkpoint', '{epoch}-{val_loss:.4f}-{val_acc:.4f}'), save_top_k=1, save_last=False, monitor="val_acc", mode="max", ) logger = TensorBoardLogger(save_dir=args.save_path, name='log') # save args with open(os.path.join(args.save_path, 'checkpoint', "args.json"), 'w') as f: args_dict = args.__dict__ del args_dict['tpu_cores'] json.dump(args_dict, f, indent=4) trainer = Trainer.from_argparse_args( args, checkpoint_callback=checkpoint_callback, distributed_backend="ddp", logger=logger) trainer.fit(model)
def test_wandb_sanitize_callable_params(tmpdir): """ Callback function are not serializiable. Therefore, we get them a chance to return something and if the returned type is not accepted, return None. """ opt = "--max_epochs 1".split(" ") parser = ArgumentParser() parser = Trainer.add_argparse_args(parent_parser=parser) params = parser.parse_args(opt) def return_something(): return "something" params.something = return_something def wrapper_something(): return return_something params.wrapper_something_wo_name = lambda: lambda: '1' params.wrapper_something = wrapper_something assert isinstance(params.gpus, types.FunctionType) params = WandbLogger._convert_params(params) params = WandbLogger._flatten_dict(params) params = WandbLogger._sanitize_callable_params(params) assert params["gpus"] == '_gpus_arg_default' assert params["something"] == "something" assert params["wrapper_something"] == "wrapper_something" assert params["wrapper_something_wo_name"] == "<lambda>"
def cli_main(args=None): from pl_bolts.callbacks import LatentDimInterpolator, TensorboardGenerativeModelImageSampler from pl_bolts.datamodules import CIFAR10DataModule, ImagenetDataModule, MNISTDataModule, STL10DataModule seed_everything(1234) parser = ArgumentParser() parser.add_argument("--dataset", default="mnist", type=str, help="mnist, cifar10, stl10, imagenet") script_args, _ = parser.parse_known_args(args) if script_args.dataset == "mnist": dm_cls = MNISTDataModule elif script_args.dataset == "cifar10": dm_cls = CIFAR10DataModule elif script_args.dataset == "stl10": dm_cls = STL10DataModule elif script_args.dataset == "imagenet": dm_cls = ImagenetDataModule parser = dm_cls.add_argparse_args(parser) parser = Trainer.add_argparse_args(parser) parser = GAN.add_model_specific_args(parser) args = parser.parse_args(args) dm = dm_cls.from_argparse_args(args) model = GAN(*dm.size(), **vars(args)) callbacks = [TensorboardGenerativeModelImageSampler(), LatentDimInterpolator(interpolate_epoch_interval=5)] trainer = Trainer.from_argparse_args(args, callbacks=callbacks, progress_bar_refresh_rate=20) trainer.fit(model, datamodule=dm) return dm, model, trainer
def main(): parser = ArgumentParser() parser = FDKConvNet.add_model_specific_args(parser) parser = Trainer.add_argparse_args(parser) hparams = parser.parse_args() hparams.lr = 5e-2 hparams.end_lr = 1e-2 hparams.max_epochs = 300 hparams.batch_size = CONFIGURATION['batch_size'] hparams.data_dir = DATA_DIRS['datasets'] hparams.valid_dir = 'valid_fdkconvnet' \ + ('_pre' if hparams.pretrained else '') with open('train_valid.json') as json_file: json_dict = json.load(json_file) hparams.train_files = json_dict['train_files'] hparams.valid_files = json_dict['valid_files'] model = FDKConvNet(**vars(hparams)) checkpoint_callback = ModelCheckpoint( dirpath=hparams.valid_dir, monitor='val_loss', save_last=True, ) trainer = Trainer( precision=CONFIGURATION['precision'], progress_bar_refresh_rate=CONFIGURATION['progress_bar_refresh_rate'], gpus=1, checkpoint_callback=checkpoint_callback, max_epochs=hparams.max_epochs, terminate_on_nan=True, ) trainer.fit(model)
def main(): parser = ArgumentParser() # add task level args parser = add_task_specific_args(parser) # add model specific args parser = Model.add_model_specific_args(parser) parser = optimization.add_optimizer_specific_args(parser) parser = Trainer.add_argparse_args(parser) # set default args parser.set_defaults(num_labels=27) args = parser.parse_args() if args.build_dataset: build_distill_dataset(args) else: common_train( args, metric=f'val_{task_info.metric_name}', model_class=Model, build_method=build_method, task=task_info.task_name )
def main(): parser = get_parser() parser = BertForGLUETask.add_model_specific_args(parser) parser = Trainer.add_argparse_args(parser) args = parser.parse_args() task_model = BertForGLUETask(args) if len(args.pretrained_checkpoint) > 1: task_model.load_state_dict( torch.load(args.pretrained_checkpoint, map_location=torch.device("cpu"))["state_dict"]) checkpoint_callback = ModelCheckpoint(filepath=args.output_dir, save_top_k=args.max_keep_ckpt, save_last=False, monitor="val_f1", verbose=True, mode='max', period=-1) task_trainer = Trainer.from_argparse_args( args, checkpoint_callback=checkpoint_callback, deterministic=True) task_trainer.fit(task_model) # after training, use the model checkpoint which achieves the best f1 score on dev set to compute the f1 on test set. best_f1_on_dev, path_to_best_checkpoint = find_best_checkpoint_on_dev( args.output_dir, only_keep_the_best_ckpt=args.only_keep_the_best_ckpt_after_training) task_model.result_logger.info("=&" * 20) task_model.result_logger.info(f"Best F1 on DEV is {best_f1_on_dev}") task_model.result_logger.info( f"Best checkpoint on DEV set is {path_to_best_checkpoint}") task_model.result_logger.info("=&" * 20)
def main(): """main""" parser = get_parser() # add model specific args parser = BertLabeling.add_model_specific_args(parser) # add all the available trainer options to argparse # ie: now --gpus --num_nodes ... --fast_dev_run all work in the cli parser = Trainer.add_argparse_args(parser) args = parser.parse_args() model = BertLabeling(args) if args.pretrained_checkpoint: model.load_state_dict( torch.load(args.pretrained_checkpoint, map_location=torch.device('cpu'))["state_dict"]) checkpoint_callback = ModelCheckpoint( filepath=args.default_root_dir, save_top_k=10, verbose=True, monitor="span_f1", period=-1, mode="max", ) trainer = Trainer.from_argparse_args( args, checkpoint_callback=checkpoint_callback) trainer.fit(model)
def run_dataloader(): """test dataloader""" parser = get_parser() # add model specific args parser = BertLabeling.add_model_specific_args(parser) # add all the available trainer options to argparse # ie: now --gpus --num_nodes ... --fast_dev_run all work in the cli parser = Trainer.add_argparse_args(parser) args = parser.parse_args() args.workers = 0 args.default_root_dir = "/mnt/data/mrc/train_logs/debug" model = BertLabeling(args) from tokenizers import BertWordPieceTokenizer tokenizer = BertWordPieceTokenizer( os.path.join(args.bert_config_dir, "vocab.txt")) loader = model.get_dataloader("dev", limit=1000) for d in loader: input_ids = d[0][0].tolist() match_labels = d[-1][0] start_positions, end_positions = torch.where(match_labels > 0) start_positions = start_positions.tolist() end_positions = end_positions.tolist() if not start_positions: continue print("=" * 20) print(tokenizer.decode(input_ids, skip_special_tokens=False)) for start, end in zip(start_positions, end_positions): print(tokenizer.decode(input_ids[start:end + 1]))
def main(): parser = ArgumentParser() # add task level args parser = add_task_specific_args(parser) # add model specific args parser = Model.add_model_specific_args(parser) parser = optimization.add_optimizer_specific_args(parser) parser = Trainer.add_argparse_args(parser) # task specific default args parser.set_defaults(gradient_clip_val=1.0, min_epochs=1, max_epochs=10) parser.set_defaults(num_labels=56, arc_hidden_size=600, rel_hidden_size=600) args = parser.parse_args() if args.build_dataset: build_distill_dataset(args) elif args.tune: tune_train(args, model_class=Model, task_info=task_info, model_kwargs={'loss_func': sdp_loss}) else: common_train(args, model_class=Model, task_info=task_info, model_kwargs={'loss_func': sdp_loss})
def main(): parser = ArgumentParser() # add task level args parser = add_task_specific_args(parser) # add model specific args parser = Model.add_model_specific_args(parser) parser = optimization.add_optimizer_specific_args(parser) parser = Trainer.add_argparse_args(parser) # task specific default args parser.set_defaults(num_labels=56) parser.set_defaults(arc_hidden_size=600, rel_hidden_size=600) args = parser.parse_args() if args.build_dataset: build_distill_dataset(args) else: common_train(args, metric=f'val_{task_info.metric_name}', model_class=Model, build_method=build_method, task='sdp', loss_func=sdp_loss)
def test_sac(): """Smoke test that the SAC model runs.""" parent_parser = argparse.ArgumentParser(add_help=False) parent_parser = Trainer.add_argparse_args(parent_parser) parent_parser = SAC.add_model_specific_args(parent_parser) args_list = [ "--warm_start_size", "100", "--gpus", "0", "--env", "Pendulum-v0", "--batch_size", "10", ] hparams = parent_parser.parse_args(args_list) trainer = Trainer( gpus=hparams.gpus, max_steps=100, max_epochs= 100, # Set this as the same as max steps to ensure that it doesn't stop early val_check_interval= 1, # This just needs 'some' value, does not effect training right now fast_dev_run=True, ) model = SAC(**hparams.__dict__) trainer.fit(model)
def main(): """main""" parser = get_parser() # add model specific arguments. parser = BertForQA.add_model_specific_args(parser) # add all the available trainer options to argparse parser = Trainer.add_argparse_args(parser) args = parser.parse_args() model = BertForQA(args) if len(args.pretrained_checkpoint) > 1: model.load_state_dict(torch.load(args.pretrained_checkpoint, map_location=torch.device('cpu'))["state_dict"]) checkpoint_callback = ModelCheckpoint( filepath=args.output_dir, save_top_k=args.max_keep_ckpt, verbose=True, period=-1, mode="auto" ) trainer = Trainer.from_argparse_args( args, checkpoint_callback=checkpoint_callback, deterministic=True ) trainer.fit(model)
def test_argparse_args_parsing_fast_dev_run(cli_args, expected): """Test multi type argument with bool.""" cli_args = cli_args.split(" ") if cli_args else [] with mock.patch("argparse._sys.argv", ["any.py"] + cli_args): parser = ArgumentParser(add_help=False) parser = Trainer.add_argparse_args(parent_parser=parser) args = Trainer.parse_argparser(parser) assert args.fast_dev_run is expected
def main(): parser = add_model_specific_args() parser = Trainer.add_argparse_args(parser) args = parser.parse_args() if args.mode == 'label': label_unlabeled_data(args) else: train_model(args)
def parse_args(argv=None): argv = argv or [] parser = ArgumentParser() parser = ConvVAE.add_model_specific_args(parser) parser = Trainer.add_argparse_args(parser) args = parser.parse_args(argv) return args
def build_args(): # ------------------------ # TRAINING ARGUMENTS # ------------------------ path_config = "ixi_config.yaml" with open(path_config) as f: data = yaml.load(f, Loader=yaml.FullLoader) ixi_args = SimpleNamespace(**data) ixi_args.mask_path = ('./masks_mei/1D-Cartesian_6X_256.mat') data_path = data['data_dir'] logdir = data['output_dir'] + "/dense_edsr/ixi/edsr_transformer" # parent_parser = ArgumentParser(add_help=False) parser = UnetModule.add_model_specific_args(parent_parser) parser = Trainer.add_argparse_args(parser) num_gpus = 1 backend = "ddp" batch_size = 8 if backend == "ddp" else num_gpus # module config config = dict( n_channels_in=1, n_channels_out=1, n_resgroups=5, # 10 n_resblocks=8, # 20 n_feats=64, # 64 lr=0.00005, lr_step_size=40, lr_gamma=0.1, weight_decay=0.0, data_path=data_path, exp_dir=logdir, exp_name="unet_demo", test_split="test", batch_size=batch_size, ixi_args=ixi_args, ) parser.set_defaults(**config) # trainer config parser.set_defaults( gpus=num_gpus, max_epochs=50, default_root_dir=logdir, replace_sampler_ddp=(backend != "ddp"), distributed_backend=backend, seed=42, deterministic=True, ) parser.add_argument("--mode", default="train", type=str) args = parser.parse_args() return args
def cli_main(): parser = ArgumentParser() parser = Trainer.add_argparse_args(parser) parser = AE.add_model_specific_args(parser) args = parser.parse_args() ae = AE(**vars(args)) trainer = Trainer.from_argparse_args(args) trainer.fit(ae)
def _parse_args(self) -> Dict: parser = argparse.ArgumentParser() parser = Trainer.add_argparse_args(parser) parser = Workspace.add_argparse_args(parser) parser = self.pl_module_cls.add_argparse_args(parser) args = parser.parse_args() return args.__dict__
def init_trainer(): """ Init a Lightning Trainer using from_argparse_args Thus every CLI command (--gpus, distributed_backend, ...) become available. """ parser = ArgumentParser() parser = Trainer.add_argparse_args(parser) args = parser.parse_args() lr_logger = LearningRateLogger() return Trainer.from_argparse_args(args, callbacks=[lr_logger])
def get_args(): parser = ArgumentParser() parser = Trainer.add_argparse_args(parser) temp_args, _ = parser.parse_known_args() file = open('project/ecgresnet_config.json', 'r') ECGResNet_params = json.load(file) file.close() return parser, ECGResNet_params