def add_model_specific_args(parent_parser, root_dir):
"""
parameters defined here will be available to the model through self.hparams
"""
# fmt: off
parser = HyperOptArgumentParser(parents=[parent_parser])
parser.opt_list('--bs', default=64, type=int, options=[32, 128, 256], tunable=True,
help='mini-batch size (default: 256), this is the total batch size of all GPUs'
'on the current node when using Data Parallel or Distributed Data Parallel')
parser.opt_list('--projection_size', default=256, type=int, options=[32, 128, 512], tunable=True,
help='sentence embedding size and hidden size for the second transformer')
parser.opt_list('--n_layers', default=1, type=int, options=[2, 4, 6], tunable=True,
help='number of encoder layers for the second transformer')
parser.opt_list('--frozen_epochs', default=2, type=int, options=[3, 6, 9], tunable=True,
help='number of epochs the pretrained DistilBert is frozen')
parser.opt_range('--lr', default=2.0e-5, type=float, tunable=True, low=1.0e-5, high=5.0e-4,
nb_samples=5, help='initial learning rate')
parser.opt_list('--layerwise_decay', default=0.95, type=float, options=[0.3, 0.6, 0.8], tunable=True,
help='layerwise decay factor for the learning rate of the pretrained DistilBert')
parser.opt_list('--max_seq_len', default=32, type=int, options=[16, 64], tunable=False,
help='maximal number of input tokens for the DistilBert model')
parser.opt_list('--dropout', default=0.1, type=float, options=[0.1, 0.2], tunable=False)
parser.add_argument('--train_file', default=os.path.join(root_dir, 'data/clean_train.txt'), type=str)
parser.add_argument('--val_file', default=os.path.join(root_dir, 'data/clean_val.txt'), type=str)
parser.add_argument('--test_file', default=os.path.join(root_dir, 'data/clean_test.txt'), type=str)
parser.add_argument('--epochs', default=3, type=int, metavar='N',
help='number of total epochs to run')
parser.add_argument('--seed', type=int, default=None,
help='seed for initializing training')
# fmt: on
return parser
def argParser():
"""
This function creates a parser object which parses all the flags from the command line
We can access the parsed command line values using the args object returned by this function
Usage:
First field is the flag name.
dest=NAME is the name to reference when using the parameter (args.NAME)
default is the default value of the parameter
Example:
> python run.py --batch-size 100
args.batch_size <-- 100
"""
# parser = argparse.ArgumentParser()
parser = HyperOptArgumentParser(strategy='random_search')
# trainer arguments
parser.add_argument("--gpu", dest="gpu", default='0', help="GPU number")
parser.add_argument("--mode", dest="mode", default='train', help="Mode is one of 'train', 'test'")
parser.add_argument("--encode", dest="encode", default=0, type=int, help="encode is 0 or 1, default 0")
parser.add_argument("--ntrials", dest="ntrials", default=20, type=int, help="Number of trials to run for hyperparameter tuning")
# model-specific arguments
# (non-tunable)
parser.add_argument("--model", dest="model", default="baseline_lstm", help="Name of model to use")
parser.add_argument("--epochs", dest="epochs", type=int, default=10, help="Number of epochs to train for")
parser.add_argument("--patience", dest="patience", type=int, default=10, help="Learning rate decay scheduler patience, number of epochs")
# (tunable arguments)
parser.opt_list("--batch-size", dest="batch_size", type=int, default=100, help="Size of the minibatch",
tunable=False, options=[32, 64, 128, 256])
parser.opt_range("--learning-rate", dest="learning_rate", type=float, default=1e-3, help="Learning rate for training",
tunable=True, low=1e-3, high=1e-1, nb_samples=4)
parser.opt_list("--hidden-size", dest="hidden_size", type=int, default=100, help="Dimension of hidden layers",
tunable=False, options=[32, 64, 128, 256])
parser.opt_list('--optimizer', dest="optimizer", type=str, default='SGD', help='Optimizer to use (default: SGD)',
tunable=False, options=['SGD', 'Adam'])
parser.opt_range('--weight-decay', dest="weight_decay", type=float, default=1e-5,
help='Weight decay for L2 regularization.',
tunable=True, low=1e-6, high=1e-1, nb_samples=10)
parser.opt_list('--frame-freq', dest="frame_freq", type=int, default=5,
help='Frequency for sub-sampling frames from a video',
tunable=True, options=[10, 30, 60, 75, 100])
# (tcn-only arguments)
parser.opt_list('--dropout', dest="dropout", type=float, default=0.05, help='Dropout applied to layers (default: 0.05)',
tunable=True, options=[0.05, 0.1, 0.3, 0.5, 0.7])
parser.opt_list('--levels', dest="levels", type=int, default=8, help='# of levels for TCN (default: 8)',
tunable=True, options=[6, 8, 10, 12])
# LSTM only arguments
parser.opt_list('--num_layers', dest="num_layers", type=int, default=1, help='# of layers in LSTM (default:1',
tunable=True, options=[1, 2, 3, 4, 5])
# program arguments (dataset and logger paths)
parser.add_argument("--raw_data_path", dest="raw_data_path", default="/mnt/disks/disk1/raw", help="Path to raw dataset")
parser.add_argument('--proc_data_path', dest="proc_data_path", default="/mnt/disks/disk1/processed", help="Path to processed dataset")
parser.add_argument("--log", dest="log", default='', help="Unique log directory name under log/. If the name is empty, do not store logs")
parser.add_argument("--checkpoint", dest="checkpoint", type=str, default="", help="Path to the .pth checkpoint file. Used to continue training from checkpoint")
# create argparser
args = parser.parse_args()
return args
def main():
parser = HyperOptArgumentParser(
description='Train a PyTorch Lightning model on Yest dataset',
strategy='random_search'
)
parser.opt_list('--nb_layers', default=2, type=int, tunable=False, options=[2, 4, 8])
parser.opt_range('--layer_size', default=20, type=int, tunable=False, low=10, high=200, nb_samples=10, help="size of the hidden layer")
parser.add_argument('--model', default="model.ptl", help="path to save the model")
parser.add_argument('--train', default="yeast_train.svm", help="path to the training data")
parser.add_argument('--val', default="yeast_test.svm", help="path to the training data")
hparams = parser.parse_args()
hparams.optimize_parallel_cpu(train_main, nb_trials=20, nb_workers=8)
if __name__ == '__main__':
usage_str = 'usage: %(prog)s [options]'
description_str = 'deep inverse problems optimization'
parser = HyperOptArgumentParser(
usage=usage_str,
description=description_str,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
strategy='random_search')
parser.opt_range('--step',
type=float,
dest='step',
default=.001,
help='step size/learning rate',
tunable=True,
nb_samples=100,
low=.0001,
high=.001)
parser.opt_range('--l2lam_init',
action='store',
type=float,
dest='l2lam_init',
default=.001,
tunable=False,
low=.0001,
high=100,
help='initial l2 regularization')
parser.opt_list('--solver',
action='store',
print(auc)
# save Experiment
exp.add_scalar('auc', auc)
exp.save()
if __name__ == '__main__':
# Set up our argparser and make the y_val tunable.
parser = HyperOptArgumentParser(strategy='random_search')
parser.add_argument('--test_tube_exp_name', default='tt_exp')
parser.add_argument('--batch_size', default=80, type=int)
parser.add_argument('--epochs', default=3, type=int)
parser.add_argument('--gpus', default=2, type=int)
parser.add_argument('--log_path', default=os.environ['SCRATCH'] + '/summer_school/hs_log')
parser.opt_range('--lr', default=2e-4, type=float, tunable=True, low=1e-4, high=1e-3, nb_samples=100, log_base=10)
parser.opt_range('--wd', default=1e-5, type=float, tunable=True, low=1e-7, high=1e-4, nb_samples=100, log_base=10)
hyperparams = parser.parse_args()
# Enable cluster training.
cluster = SlurmCluster(
hyperparam_optimizer=hyperparams,
log_path=hyperparams.log_path,
python_cmd='python3',
)
# Email results if your hpc supports it.
# cluster.notify_job_statusi(email='*****@*****.**', on_done=True, on_fail=True)
# SLURM Module to load.
cluster.load_modules([
