def add_optim_specific_args( parser: HyperOptArgumentParser, ) -> HyperOptArgumentParser: """ Functions that parses Optimizer specific arguments and adds them to the Namespace :param parent_parser: """ parser = super(RAdam, RAdam).add_optim_specific_args(parser) parser.add_argument("--b1", default=0.9, type=float, help="Adams beta parameters (b1, b2).") parser.add_argument("--b2", default=0.999, type=float, help="Adams beta parameters (b1, b2).") parser.add_argument("--eps", default=1e-6, type=float, help="Adams epsilon.") parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay.") parser.add_argument( "--degenerated_to_sgd", default=False, help= "If this flag is on the degenerated_to_sgd RAdam parameter is set to True.", action="store_true", ) return parser
def add_optim_specific_args( parser: HyperOptArgumentParser, ) -> HyperOptArgumentParser: """ Functions that parses Optimizer specific arguments and adds them to the Namespace :param parser: """ parser = super(Adam, Adam).add_optim_specific_args(parser) parser.add_argument("--b1", default=0.9, type=float, help="Adams beta parameters (b1, b2).") parser.add_argument("--b2", default=0.999, type=float, help="Adams beta parameters (b1, b2).") parser.add_argument("--eps", default=1e-6, type=float, help="Adams epsilon.") parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay.") parser.add_argument( "--amsgrad", default=False, help= "Whether to use the AMSGrad variant of this algorithm from the paper:\ 'On the Convergence of Adam and Beyond'", action="store_true", ) return parser
def add_model_specific_args(parent: HyperOptArgumentParser, root_dir): parser = HyperOptArgumentParser(strategy=parent.strategy, parents=[parent]) parser.add_argument( "--data_root", default=os.path.join(root_dir, "livedoor-news-corpus", "text"), ) parser.add_argument("--max_vocab", default=30000, type=int) parser.add_argument("--latent_dim", default=80, type=int) parser.add_argument("--encoder_num_layers", default=2, type=int) return parser
def add_scheduler_specific_args( parser: HyperOptArgumentParser, ) -> HyperOptArgumentParser: """ Functions that parses scheduler specific arguments and adds them to the Namespace :param parser: """ parser.add_argument("--last_epoch", default=-1, type=int, help="Scheduler last epoch step") return parser
def main(): parent_parser = HyperOptArgumentParser(strategy="grid_search", add_help=False) logdir = "logs" parent_parser.add_argument( "--test_tube_save_path", default=os.path.join(logdir, "test_tube_data") ) parent_parser.add_argument( "--model_save_path", default=os.path.join(logdir, "model_weights") ) parent_parser.add_argument( "--experiment_name", default=os.path.join(logdir, "vampire") ) parser = VAMPIRE.add_model_specific_args(parent_parser, ".") hparams = parser.parse_args() model = VAMPIRE(hparams) exp = Experiment( name=hparams.experiment_name, save_dir=hparams.test_tube_save_path, autosave=False, ) exp.argparse(hparams) exp.save() trainer = Trainer(experiment=exp, fast_dev_run=False) trainer.fit(model)
def add_optim_specific_args( parser: HyperOptArgumentParser, ) -> HyperOptArgumentParser: """ Functions that parses Optimizer specific arguments and adds them to the Namespace :param parser: """ parser = super(Adamax, Adamax).add_optim_specific_args(parser) parser.add_argument("--b1", default=0.9, type=float, help="Adams beta parameters (b1, b2).") parser.add_argument("--b2", default=0.999, type=float, help="Adams beta parameters (b1, b2).") parser.add_argument("--eps", default=1e-6, type=float, help="Adams epsilon.") parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay.") return parser
def test_preprocessing_noise(self): dataset = MaestroDataset('/Volumes/Elements/Datasets/maestro-v2.0.0', item_length=176400, sampling_rate=44100, mode='validation', max_file_count=20, shuffle_with_seed=123) parser = HyperOptArgumentParser(strategy='random_search', add_help=False) #parser = ArgumentParser() parser = ContrastivePredictiveSystem.add_model_specific_args( parser, root_dir='../') hparams = parser.parse_args() preprocessing = PreprocessingModule(hparams) pass
def main(force_test=False): mp.set_start_method("spawn", force=True) parser = (argparse.ArgumentParser(description='Order embedding arguments') if not HYPERPARAM_SEARCH else HyperOptArgumentParser( strategy='grid_search')) utils.parse_optimizer(parser) parse_encoder(parser) args = parser.parse_args() print(args) args.n_workers = 1 if force_test: args.test = True # Currently due to parallelism in multi-gpu training, this code performs # sequential hyperparameter tuning. # All gpus are used for every run of training in hyperparameter search. if HYPERPARAM_SEARCH: for i, hparam_trial in enumerate( args.trials(HYPERPARAM_SEARCH_N_TRIALS)): print("Running hyperparameter search trial", i) print(hparam_trial) train_loop(hparam_trial) else: train_loop(args)
def add_optim_specific_args( parser: HyperOptArgumentParser, ) -> HyperOptArgumentParser: """ Functions that parses Optimizer specific arguments and adds them to the Namespace :param parser: """ parser.opt_list( "--learning_rate", default=5e-5, type=float, tunable=True, options=[1e-05, 3e-05, 5e-05, 8e-05, 1e-04], help="Optimizer learning rate.", ) return parser
def test_add_to_parser(): parser = HyperOptArgumentParser(strategy='grid_search') utils.add_to_parser(parser, 'test0', '4') utils.add_to_parser(parser, 'test1', '5') utils.add_to_parser(parser, 'test2', [1, 2, 3]) namespace, _ = parser.parse_known_args(['--test0', '3']) # single argument assert namespace.test0 == '3' # user defined arg assert namespace.test1 == '5' # default arg # list argument assert namespace.test2 is None assert parser.opt_args['--test2'].opt_values == [1, 2, 3] assert parser.opt_args['--test2'].tunable
def add_scheduler_specific_args( parser: HyperOptArgumentParser, ) -> HyperOptArgumentParser: """ Functions that parses Optimizer specific arguments and adds them to the Namespace :param parent_parser: """ parser = super(WarmupConstant, WarmupConstant).add_scheduler_specific_args(parser) parser.add_argument( "--warmup_steps", type=int, default=1, help= "Linearly increases learning rate from 0*learning_rate to 1*learning_rate over warmup_steps.", ) return parser
def get_args(): parser = argparse.ArgumentParser() parser.add_argument('-s', '--sweep', action='store_true', help='Run a hyperparameter sweep over all options') # DataModule args parser = MNISTDataModule.add_argparse_args(parser) # Trainer args (https://pytorch-lightning.readthedocs.io/en/latest/common/trainer.html#trainer-flags) parser = Trainer.add_argparse_args(parser) # Set some sane defaults for x in parser._actions: if x.dest == 'gpus': x.default = 1 if x.dest == 'max_epochs': x.default = 100 # TestTube args - hyperparam parser & slurm info parser = HyperOptArgumentParser(strategy='grid_search', add_help=False, parents=[parser]) parser.add_argument('--test_tube_exp_name', default='sweep_test') parser.add_argument('--log_path', default='./pytorch-slurm') # LightningModule args (hyperparameters) parser = MNISTClassifier.add_model_specific_args(parser) args = parser.parse_args() return args
def add_model_specific_args(parent_parser): parser = HyperOptArgumentParser(parents=[parent_parser], add_help=False) # want to optimize this parameter #parser.opt_list('--batch_size', type=int, default=16, options=[16, 10, 8], tunable=False) parser.opt_list('--learning_rate', type=float, default=0.001, options=[1e-3, 1e-4, 1e-5], tunable=True) parser.add_argument('--batch_size', type=int, default=10) # fixed arguments parser.add_argument('--output_img_freq', type=int, default=100) parser.add_argument('--unfreeze_epoch_no', type=int, default=0) parser.add_argument('--mse_loss', default=False, action='store_true') return parser
def get_default_parser(strategy, root_dir): possible_model_names = list(AVAILABLE_MODELS.keys()) parser = HyperOptArgumentParser(strategy=strategy, add_help=False) add_default_args(parser, root_dir, possible_model_names=possible_model_names, rand_seed=SEED) return parser
def hparams(self): parser = HyperOptArgumentParser() # metric mode and monitor are hparams required by COMET models # and lightning trainer. parser.add_argument("--monitor", default="slot_error_rate") parser.add_argument("--metric_mode", default="min") parser = TransformerTagger.add_model_specific_args(parser) hparams, _ = parser.parse_known_args([]) return hparams
def add_data_args(parser: HyperOptArgumentParser) -> HyperOptArgumentParser: """ Functions that parses dataset specific arguments/hyperparameters. :param hparams: HyperOptArgumentParser obj. Returns: - updated parser """ parser.add_argument( "--data_type", default="csv", type=str, help="The type of the file containing the training/dev/test data.", choices=["csv"], ) parser.add_argument( "--train_path", default="data/dummy_train.csv", type=str, help="Path to the file containing the train data.", ) parser.add_argument( "--dev_path", default="data/dummy_test.csv", type=str, help="Path to the file containing the dev data.", ) parser.add_argument( "--test_path", default="data/dummy_test.csv", type=str, help="Path to the file containing the test data.", ) parser.add_argument( "--loader_workers", default=0, type=int, help="How many subprocesses to use for data loading. 0 means that \ the data will be loaded in the main process.", ) return parser
def add_model_specific_args( cls, parser: HyperOptArgumentParser ) -> HyperOptArgumentParser: """ Parser for Estimator specific arguments/hyperparameters. :param parser: HyperOptArgumentParser obj Returns: - updated parser """ parser.add_argument( "--learning_rate", default=3e-05, type=float, help="Learning rate.", ) # Data Args: parser.add_argument( "--train_csv", default="data/train_data.csv", type=str, help="Path to the file containing the train data.", ) parser.add_argument( "--dev_csv", default="data/valid_data.csv", type=str, help="Path to the file containing the dev data.", ) parser.add_argument( "--test_csv", default="data/valid_data.csv", type=str, help="Path to the file containing the dev data.", ) parser.add_argument( "--loader_workers", default=8, type=int, help="How many subprocesses to use for data loading. 0 means that \ the data will be loaded in the main process.", ) return parser
def add_scheduler_specific_args( parser: HyperOptArgumentParser, ) -> HyperOptArgumentParser: """ Functions that parses Optimizer specific arguments and adds them to the Namespace :param parent_parser: """ parser = super(LinearWarmup, LinearWarmup).add_scheduler_specific_args(parser) parser.add_argument( "--warmup_steps", type=int, default=1, help="Linearly increases learning rate from 0 to 1 over warmup_steps.", ) parser.add_argument( "--num_training_steps", type=int, default=sys.maxsize, help="Linearly decreases learning rate from 1*learning_rate to 0*learning_rate over \ remaining t_total - warmup_steps steps.", ) return parser
def load_yaml_args(parser: HyperOptArgumentParser, log): """ Function that load the args defined in a YAML file and replaces the values parsed by the HyperOptArgumentParser """ old_args = vars(parser.parse_args()) configs = old_args.get("config") if configs: yaml_file = yaml.load(open(configs).read(), Loader=yaml.FullLoader) for key, value in yaml_file.items(): if key in old_args: old_args[key] = value else: raise Exception( "{} argument defined in {} is not valid!".format( key, configs)) else: log.warning("We recommend the usage of YAML files to keep track \ of the hyperparameter during testing and training.") return TTNamespace(**old_args)
def get_all_params(): # raise error if user has other command line arguments specified if len(sys.argv[1:]) != 6: raise ValueError( 'No command line arguments allowed other than config file names') def add_to_parser(parser, arg_name, value): if arg_name == 'expt_ids' or arg_name == 'expt_ids_to_keep': # treat expt_ids differently, want to parse full lists as one if isinstance(value, list): value = ';'.join(value) parser.add_argument('--' + arg_name, default=value) elif isinstance(value, list): parser.opt_list('--' + arg_name, options=value, tunable=True) else: parser.add_argument('--' + arg_name, default=value) # create parser parser = HyperOptArgumentParser(strategy='grid_search') parser.add_argument('--data_config', type=str) parser.add_argument('--model_config', type=str) parser.add_argument('--train_config', type=str) namespace, extra = parser.parse_known_args() # add arguments from all configs configs = [ namespace.data_config, namespace.model_config, namespace.train_config ] for config in configs: config_dict = yaml.safe_load(open(config)) for (key, value) in config_dict.items(): add_to_parser(parser, key, value) return parser.parse_args()
def add_model_specific_args(parent_parser, root_dir): # pragma: no cover """ Parameters you define here will be available to your model through self.hparams :param parent_parser: :param root_dir: :return: """ parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.add_argument('--in_features', default=28 * 28, type=int) parser.add_argument('--out_features', default=10, type=int) parser.add_argument( '--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001 * 8, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu parser.opt_list( '--batch_size', default=256 * 8, type=int, options=[32, 64, 128, 256], tunable=False, help= 'batch size will be divided over all the gpus being used across all nodes' ) return parser
cluster.optimize_parallel_cluster_gpu(main, nb_trials=hyperparams.nb_hopt_trials, job_name=hyperparams.experiment_name) if __name__ == '__main__': # use default args root_dir = os.path.dirname(os.path.realpath(__file__)) demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs') checkpoint_dir = os.path.join(demo_log_dir, 'model_weights') test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data') slurm_out_dir = os.path.join(demo_log_dir, 'slurm_scripts') parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False) # cluster args not defined inside the model parent_parser.add_argument('--gpu_partition', type=str, help='consult your cluster manual') # TODO: make 1 param parent_parser.add_argument('--per_experiment_nb_gpus', type=int, default=2, help='how many gpus to use in a node') parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node')
from model import GNNModel from test_tube import Experiment from test_tube import HyperOptArgumentParser from pytorch_lightning.callbacks import ModelCheckpoint import os from pytorch_lightning.callbacks import EarlyStopping from pytorch_lightning.logging import TestTubeLogger import uuid from datetime import datetime import random if __name__ == '__main__': parser = HyperOptArgumentParser() # runtime params parser.add_argument('--dataset', type=str, choices=['cora','citeseer','pubmed','amazon'], default='cora') parser.add_argument('--percentage', type=float) parser.add_argument('--log-dir', type=str, default='log') parser.add_argument('--ckpt-name', type=str) parser.add_argument('--seed', type=int, default=2020) parser.add_argument('--opt', default='adam', choices=['adam','lbfgs']) parser.add_argument("--gpus", type=str, default='0', help="gpu") parser.add_argument("--lr", type=float, default=1e-2, help="learning rate") parser.add_argument("--weight-decay", type=float, default=5e-6, help="Weight for L2 loss") parser.add_argument("--n-epochs", type=int, default=200, help="number of training epochs")
def add_model_specific_args(parent_parser, root_dir): # pragma: no cover """ Parameters you define here will be available to your model through self.hparams """ parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--local', default=3, options=[3, 5, 7], type=int, tunable=True) parser.opt_list('--n_kernels', default=32, options=[32, 50, 100], type=int, tunable=True) parser.add_argument('-w_kernel', type=int, default=1) parser.opt_list('--d_model', type=int, default=512, options=[512], tunable=False) parser.opt_list('--d_inner', type=int, default=2048, options=[2048], tunable=False) parser.opt_list('--d_k', type=int, default=64, options=[64], tunable=False) parser.opt_list('--d_v', type=int, default=64, options=[64], tunable=False) parser.opt_list('--n_head', type=int, default=8, options=[8], tunable=False) parser.opt_list('--n_layers', type=int, default=6, options=[6], tunable=False) parser.opt_list('--drop_prob', type=float, default=0.1, options=[0.1, 0.2, 0.5], tunable=False) # arguments from dataset parser.add_argument('--data_name', type=str) parser.add_argument('--data_dir', default='.\data', type=str) parser.add_argument('--n_multiv', type=int) parser.opt_list('--window', default=64, type=int, options=[32, 64, 128], tunable=True) parser.opt_list('--horizon', default=3, type=int, options=[3, 6, 12, 24], tunable=True) # training params (opt) parser.opt_list('--learning_rate', default=0.005, type=float, options=[0.0001, 0.0005, 0.001, 0.005, 0.008], tunable=True) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) parser.opt_list('--criterion', default='mse_loss', type=str, options=['l1_loss', 'mse_loss'], tunable=False) # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu parser.opt_list( '--batch_size', default=16, type=int, options=[16, 32, 64, 128, 256], tunable=False, help= 'batch size will be divided over all the gpus being used across all nodes' ) return parser
# ------------------------ # 4 START TRAINING # ------------------------ trainer.fit(model) if __name__ == '__main__': # dirs root_dir = os.path.dirname(os.path.realpath(__file__)) demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs') checkpoint_dir = os.path.join(demo_log_dir, 'model_weights') test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data') # although we user hyperOptParser, we are using it only as argparse right now parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False) # gpu args parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node.' 'value -1 uses all the gpus on the node') parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs') parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
save_top_k=hparams.save_top_k, verbose=True, monitor=hparams.monitor, period=1, mode=hparams.metric_mode, ) trainer.checkpoint_callback = checkpoint_callback trainer.fit(model) if __name__ == "__main__": parser = HyperOptArgumentParser( strategy="random_search", description="Minimalist BERT Classifier", add_help=True, ) parser.add_argument("--seed", type=int, default=3, help="Training seed.") parser.add_argument( "--save_top_k", default=1, type=int, help= "The best k models according to the quantity monitored will be saved.", ) # Early Stopping parser.add_argument("--monitor", default="val_acc", type=str, help="Quantity to monitor.")
def add_model_specific_args( cls, parser: HyperOptArgumentParser) -> HyperOptArgumentParser: """ Parser for Estimator specific arguments/hyperparameters. :param parser: HyperOptArgumentParser obj Returns: - updated parser """ parser = super(Tagger, Tagger).add_model_specific_args(parser) parser.add_argument( "--tag_set", type=str, default="L,U,T", help="Task tags we want to use.\ Note that the 'default' label should appear first", ) # Loss parser.add_argument( "--loss", default="cross_entropy", type=str, help="Loss function to be used.", choices=["cross_entropy"], ) parser.add_argument( "--class_weights", default="ignore", type=str, help= 'Weights for each of the classes we want to tag (e.g: "1.0,7.0,8.0").', ) ## Data args: parser.add_argument( "--data_type", default="csv", type=str, help="The type of the file containing the training/dev/test data.", choices=["csv"], ) parser.add_argument( "--train_path", default="data/dummy_train.csv", type=str, help="Path to the file containing the train data.", ) parser.add_argument( "--dev_path", default="data/dummy_test.csv", type=str, help="Path to the file containing the dev data.", ) parser.add_argument( "--test_path", default="data/dummy_test.csv", type=str, help="Path to the file containing the test data.", ) parser.add_argument( "--loader_workers", default=0, type=int, help=("How many subprocesses to use for data loading. 0 means that" "the data will be loaded in the main process."), ) # Metric args: parser.add_argument( "--ignore_first_title", default=False, help="When used, this flag ignores T tags in the first position.", action="store_true", ) parser.add_argument( "--ignore_last_tag", default=False, help="When used, this flag ignores S tags in the last position.", action="store_true", ) return parser
import torch import torch.nn.functional as F from test_tube import Experiment, HyperOptArgumentParser, HyperParamOptimizer from torch import distributions, nn from torch.autograd import Variable from torch.nn import Linear from torch.optim import Adam from utils import ReplayBuffer, Step, np_to_var # In paper, ant was dropped into 9 envs, but its S,A were same. No transfer # learning yet. exp = Experiment("meta learning shared hierarchies", save_dir="logs") parser = HyperOptArgumentParser(strategy="random_search") parser.opt_list( "--batch_size", default=128, type=int, tunable=True, options=[2**n for n in range(5, 10)], ) args = parser.parse_args() args.max_steps = 1000 args.subpolicy_duration = 200 args.num_policies = 10 args.max_buffer_size = 1_000_000 args.env_names = ["Ant-v2"]
# distributed_backend="dp", # overfit_pct=0.01 ) trainer.fit(model) if __name__ == "__main__": SEED = 2538 torch.manual_seed(SEED) np.random.seed(SEED) # use default args given by lightning root_dir = os.getcwd() parent_parser = HyperOptArgumentParser(strategy="random_search", add_help=False) add_default_args(parent_parser, root_dir, rand_seed=SEED) parent_parser.add_argument( "--dist_backend", type=str, default="dp", help= "When using multiple GPUs set Trainer(distributed_backend=dp) (or ddp)", ) # allow model to overwrite or extend args parser = AutoregressiveFaceVAE.add_model_specific_args( parent_parser, root_dir) hyperparams = parser.parse_args() print(hyperparams) # train model
# define tensorflow graph x = tf.placeholder(dtype=tf.int32, name='x') y = tf.placeholder(dtype=tf.int32, name='y') out = x * y sess = tf.Session() # Run the tf op for train_step in range(0, 100): output = sess.run(out, feed_dict={x: hparams.x_val, y: hparams.y_val}) exp.log({'fake_err': output}) # save exp when we're done exp.save() # set up our argparser and make the y_val tunable parser = HyperOptArgumentParser(strategy='random_search') parser.add_argument('--test_tube_exp_name', default='my_test') parser.add_argument('--log_path', default='/Users/waf/Desktop/test') parser.opt_list('--y_val', default=12, options=[1, 2, 3, 4], tunable=True) parser.opt_list('--x_val', default=12, options=[20, 12, 30, 45], tunable=True) hyperparams = parser.parse_args() # optimize on 4 gpus at the same time # each gpu will get 1 experiment with a set of hyperparams hyperparams.optimize_parallel_gpu(train, gpu_ids=['1', '0', '3', '2'], nb_trials=4, nb_workers=4)