def add_model_specific_args(parent_parser, root_dir):
parser = HyperOptArgumentParser(strategy=parent_parser.strategy,
parents=[parent_parser])
# network params
parser.add_argument('--h_dim', default=32, type=int)
parser.add_argument('--truncation', default=64, type=int)
parser.add_argument('--truncation_offset', default=1, type=int)
parser.add_argument('--ratio',
default=None,
type=float,
help="ratio for early prediction")
# training params (opt)
parser.opt_list('--threhold',
default=0.5,
type=float,
options=[0.5, 0.6, 0.7, 0.8])
parser.opt_list('--learning_rate',
default=5e-5,
type=float,
options=[0.0001, 0.0005, 0.001, 0.005])
parser.opt_list('--optimizer_name',
default='adam',
type=str,
options=['adam'],
tunable=False)
parser.opt_list('--batch_size',
default=8,
type=int,
options=[2, 4, 8, 16])
return parser
def get_args(model):
"""
returns the HyperOptArgumentParser
"""
# fmt: off
parent_parser = HyperOptArgumentParser(strategy='random_search', add_help = False)
data_dir = os.getcwd()
parent_parser.add_argument('--mode', type=str, default='test',
choices=('default', 'test', 'hparams_search'),
help='supports default for train/test/val and hparams_search for a hyperparameter search')
parent_parser.add_argument('--save-path', metavar='DIR', default=os.environ['HOME'] + "/data/mlflow_experiments/mlruns", type=str,
help='path to save output')
parent_parser.add_argument('--gpus', type=str, default='0,1', help='which gpus')
parent_parser.add_argument('--distributed-backend', type=str, default='ddp', choices=('dp', 'ddp', 'ddp2'),
help='supports three options dp, ddp, ddp2')
parent_parser.add_argument('--use_16bit', dest='use_16bit', action='store_true',
help='if true uses 16 bit precision')
# debugging
parent_parser.add_argument('--fast_dev_run', dest='fast_dev_run', action='store_true',
help='debugging a full train/val/test loop')
parent_parser.add_argument('--track_grad_norm', dest='track_grad_norm', action='store_true',
help='inspect gradient norms')
parser = model.add_model_specific_args(parent_parser, data_dir)
# fmt: on
return parser
def parse_args():
def path(str):
return os.path.abspath((os.path.expanduser(str)))
parser = HyperOptArgumentParser()
parser.add_argument("--dataset", type=str, default="trec_web.1-200.asrc")
parser.add_argument("--asrc-path", type=path, default=None)
parser.add_argument("--log-path", type=path, default="robustness_log")
parser.add_argument("--test", action='store_true')
parser.add_argument("--fp16", action='store_true')
parser.add_argument("--gpu-num", type=int, default=1)
parser.add_argument("--model",
type=str,
choices=["bert", "mp", "conv_knrm", "all"],
default="all")
parser.add_argument("--exp",
type=str,
default="weight_decay",
choices=["dropout", "weight_decay"])
parser.add_argument("--saved-preprocessor",
type=path,
default="preprocessor")
parser.opt_list("--weight-decay",
type=float,
tunable=True,
options=[
0.0001, 0.001, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.12,
0.14, 0.16, 0.18, 0.20
])
args = parser.parse_args()
return args
def add_model_specific_args(parent_parser):
parser = HyperOptArgumentParser(parents=[parent_parser])
parser.add_argument('--num_freq_mask', default=2, type=int)
parser.add_argument('--num_time_mask', default=2, type=int)
parser.add_argument('--freq_mask_length', default=30, type=int)
parser.add_argument('--time_mask_length', default=20, type=int)
parser.add_argument('--feature_dim', default=400, type=int)
parser.add_argument('--model_size', default=512, type=int)
parser.add_argument('--feed_forward_size', default=2048, type=int)
parser.add_argument('--hidden_size', default=64, type=int)
parser.add_argument('--dropout', default=0.1, type=float)
parser.add_argument('--num_head', default=8, type=int)
parser.add_argument('--num_encoder_layer', default=6, type=int)
parser.add_argument('--num_decoder_layer', default=6, type=int)
parser.add_argument('--vocab_path',
default='testing_vocab_2.model',
type=str)
parser.add_argument('--max_feature_length', default=1024, type=int)
parser.add_argument('--max_token_length', default=50, type=int)
parser.add_argument('--share_weight', default=True, type=bool)
parser.add_argument('--loss_lambda', default=0.9, type=float)
parser.add_argument('--smoothing', default=0.1, type=float)
parser.add_argument('--lr', default=3e-4, type=float)
parser.add_argument('--warm_up_step', default=16000, type=int)
parser.add_argument('--factor', default=1, type=int)
parser.add_argument('--enable_spec_augment', default=True, type=bool)
parser.add_argument('--train_batch_size', default=64, type=int)
parser.add_argument('--train_loader_num_workers', default=16, type=int)
parser.add_argument('--val_batch_size', default=64, type=int)
parser.add_argument('--val_loader_num_workers', default=16, type=int)
return parser
def add_model_specific_args(parent_parser, root_dir):
"""
parameters defined here will be available to the model through self.hparams
"""
# fmt: off
parser = HyperOptArgumentParser(parents=[parent_parser])
parser.opt_list('--bs', default=64, type=int, options=[32, 128, 256], tunable=True,
help='mini-batch size (default: 256), this is the total batch size of all GPUs'
'on the current node when using Data Parallel or Distributed Data Parallel')
parser.opt_list('--projection_size', default=256, type=int, options=[32, 128, 512], tunable=True,
help='sentence embedding size and hidden size for the second transformer')
parser.opt_list('--n_layers', default=1, type=int, options=[2, 4, 6], tunable=True,
help='number of encoder layers for the second transformer')
parser.opt_list('--frozen_epochs', default=2, type=int, options=[3, 6, 9], tunable=True,
help='number of epochs the pretrained DistilBert is frozen')
parser.opt_range('--lr', default=2.0e-5, type=float, tunable=True, low=1.0e-5, high=5.0e-4,
nb_samples=5, help='initial learning rate')
parser.opt_list('--layerwise_decay', default=0.95, type=float, options=[0.3, 0.6, 0.8], tunable=True,
help='layerwise decay factor for the learning rate of the pretrained DistilBert')
parser.opt_list('--max_seq_len', default=32, type=int, options=[16, 64], tunable=False,
help='maximal number of input tokens for the DistilBert model')
parser.opt_list('--dropout', default=0.1, type=float, options=[0.1, 0.2], tunable=False)
parser.add_argument('--train_file', default=os.path.join(root_dir, 'data/clean_train.txt'), type=str)
parser.add_argument('--val_file', default=os.path.join(root_dir, 'data/clean_val.txt'), type=str)
parser.add_argument('--test_file', default=os.path.join(root_dir, 'data/clean_test.txt'), type=str)
parser.add_argument('--epochs', default=3, type=int, metavar='N',
help='number of total epochs to run')
parser.add_argument('--seed', type=int, default=None,
help='seed for initializing training')
# fmt: on
return parser
def add_model_specific_args(parent_parser):
parser = HyperOptArgumentParser(parents=[parent_parser])
parser.add_argument('--num_freq_mask', default=1, type=int)
parser.add_argument('--num_time_mask', default=1, type=int)
parser.add_argument('--freq_mask_length', default=10, type=int)
parser.add_argument('--time_mask_length', default=20, type=int)
parser.add_argument('--feature_dim', default=320, type=int)
parser.add_argument('--model_size', default=512, type=int)
parser.add_argument('--feed_forward_size', default=512, type=int)
parser.add_argument('--hidden_size', default=64, type=int)
parser.add_argument('--dropout', default=0.1, type=float)
parser.add_argument('--num_head', default=4, type=int)
parser.add_argument('--num_layer', default=4, type=int)
parser.add_argument('--vocab_path',
default='testing_vocab.model',
type=str)
parser.add_argument('--max_feature_length', default=1024, type=int)
parser.add_argument('--max_token_length', default=50, type=int)
parser.add_argument('--lr', default=3e-4, type=float)
parser.add_argument('--enable_spec_augment', default=False, type=bool)
parser.add_argument('--train_batch_size', default=16, type=int)
parser.add_argument('--train_loader_num_workers', default=4, type=int)
parser.add_argument('--val_batch_size', default=16, type=int)
parser.add_argument('--val_loader_num_workers', default=4, type=int)
return parser
def add_model_specific_args(parent_parser, root_dir): # pragma: no cover
"""
Parameters you define here will be available to your model through self.hparams
:param parent_parser:
:param root_dir:
:return:
"""
parser = HyperOptArgumentParser(
strategy=parent_parser.strategy, parents=[parent_parser]
)
parser.add_argument("--beta", default=1, type=float)
parser.add_argument("--bottleneck_size", default=10, type=int)
parser.add_argument("--group_distance_scaling", default=1, type=float)
parser.add_argument("--audio_size", default=80, type=int)
parser.add_argument("--data_dim", default=140, type=int)
parser.add_argument("--data_files", default="datafiles.json", type=str)
parser.add_argument("--data_dir", default="/data_dir", type=str)
parser.opt_list(
"--batch_size",
default=256 * 4,
type=int,
options=[32, 64, 128, 256],
tunable=False,
help="batch size will be divided over all gpus being used across all nodes",
)
parser.opt_list(
"--learning_rate",
default=0.001 * 8,
type=float,
options=[0.0001, 0.0005, 0.001],
tunable=True,
)
return parser
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument('-s',
'--sweep',
action='store_true',
help='Run a hyperparameter sweep over all options')
# DataModule args
parser = MNISTDataModule.add_argparse_args(parser)
# Trainer args (https://pytorch-lightning.readthedocs.io/en/latest/common/trainer.html#trainer-flags)
parser = Trainer.add_argparse_args(parser)
# Set some sane defaults
for x in parser._actions:
if x.dest == 'gpus':
x.default = 1
if x.dest == 'max_epochs':
x.default = 100
# TestTube args - hyperparam parser & slurm info
parser = HyperOptArgumentParser(strategy='grid_search',
add_help=False,
parents=[parser])
parser.add_argument('--test_tube_exp_name', default='sweep_test')
parser.add_argument('--log_path', default='./pytorch-slurm')
# LightningModule args (hyperparameters)
parser = MNISTClassifier.add_model_specific_args(parser)
args = parser.parse_args()
return args
def add_model_specific_args(parent_parser, root_dir):
parser = HyperOptArgumentParser(
strategy=parent_parser.strategy, parents=[parent_parser])
# network params
parser.add_argument('--h_dim', default=32, type=int)
parser.add_argument('--only_rnn', default=False, action='store_true')
parser.add_argument('--is_multi', default=False, action='store_true')
parser.add_argument('--truncation', default=64, type=int)
parser.add_argument('--truncation_offset', default=1, type=int)
parser.add_argument('--duration', default=90, type=int)
parser.add_argument('--ratio', default=None,
type=float, help="ratio for early prediction")
# training params (opt)
parser.opt_list('--heads', default=1, type=int,
options=[2, 4, 5], tunable=False)
parser.opt_list('--n_layers', default=2, type=int,
options=[2, 4, 5], tunable=False)
parser.opt_list('--threhold', default=0.5, type=float,
options=[0.3, 0.4, 0.5, 0.6], tunable=False)
parser.opt_list('--learning_rate', default=5e-5, type=float,
options=[0.0001, 0.0005, 0.001, 0.005],
tunable=False)
parser.opt_list('--optimizer_name', default='adam', type=str,
options=['adam'], tunable=False)
parser.opt_list('--batch_size', default=8, type=int)
return parser
def add_model_specific_args(parent_parser):
parser = HyperOptArgumentParser(parents=[parent_parser],
add_help=False)
# want to optimize this parameter
#parser.opt_list('--batch_size', type=int, default=16, options=[16, 10, 8], tunable=False)
parser.opt_list('--learning_rate',
type=float,
default=1e-3,
options=[1e-3, 1e-4, 1e-5],
tunable=False)
parser.opt_list('--unfreeze_epoch_no',
type=int,
default=0,
options=[0, 20],
tunable=True)
#parser.opt_list('--loss_fn', type=str, default='mse', options=['mse', 'bce'], tunable=True)
parser.add_argument('--batch_size', type=int, default=16)
# fixed arguments
parser.add_argument('--link',
type=str,
default='/scratch/ab8690/DLSP20Dataset/data')
#parser.add_argument('--pretrained_path', type=str, default='/scratch/ab8690/logs/dd_pretrain_ae/lightning_logs/version_9234267/checkpoints/epoch=42.ckpt')
parser.add_argument(
'--pretrained_path',
type=str,
default=
'/scratch/ab8690/logs/space_bb_pretrain/lightning_logs/version_9604234/checkpoints/epoch=23.ckpt'
)
parser.add_argument('--output_img_freq', type=int, default=500)
return parser
def main(force_test=False):
mp.set_start_method("spawn", force=True)
parser = (argparse.ArgumentParser(description='Order embedding arguments')
if not HYPERPARAM_SEARCH else HyperOptArgumentParser(
strategy='grid_search'))
utils.parse_optimizer(parser)
parse_encoder(parser)
args = parser.parse_args()
print(args)
args.n_workers = 1
if force_test:
args.test = True
# Currently due to parallelism in multi-gpu training, this code performs
# sequential hyperparameter tuning.
# All gpus are used for every run of training in hyperparameter search.
if HYPERPARAM_SEARCH:
for i, hparam_trial in enumerate(
args.trials(HYPERPARAM_SEARCH_N_TRIALS)):
print("Running hyperparameter search trial", i)
print(hparam_trial)
train_loop(hparam_trial)
else:
train_loop(args)
def add_model_specific_args(parent_parser):
from test_tube import HyperOptArgumentParser
parser = HyperOptArgumentParser(parents=[parent_parser],
add_help=False)
parser.add_argument('--base_encoder', type=str, default='resnet18')
parser.add_argument('--emb_dim', type=int, default=128)
parser.add_argument('--num_workers', type=int, default=8)
parser.add_argument('--num_negatives', type=int, default=65536)
parser.add_argument('--encoder_momentum', type=float, default=0.999)
parser.add_argument('--softmax_temperature', type=float, default=0.07)
parser.add_argument('--learning_rate', type=float, default=0.03)
parser.add_argument('--momentum', type=float, default=0.9)
parser.add_argument('--weight_decay', type=float, default=1e-4)
parser.add_argument('--data_dir', type=str, default='./')
parser.add_argument('--dataset',
type=str,
default='cifar10',
help='cifar10, stl10, imagenet2012')
parser.add_argument('--batch_size', type=int, default=256)
parser.add_argument('--use_mlp', action='store_true')
parser.add_argument('--meta_dir',
default='.',
type=str,
help='path to meta.bin for imagenet')
return parser
def argParser():
"""
This function creates a parser object which parses all the flags from the command line
We can access the parsed command line values using the args object returned by this function
Usage:
First field is the flag name.
dest=NAME is the name to reference when using the parameter (args.NAME)
default is the default value of the parameter
Example:
> python run.py --batch-size 100
args.batch_size <-- 100
"""
# parser = argparse.ArgumentParser()
parser = HyperOptArgumentParser(strategy='random_search')
# trainer arguments
parser.add_argument("--gpu", dest="gpu", default='0', help="GPU number")
parser.add_argument("--mode", dest="mode", default='train', help="Mode is one of 'train', 'test'")
parser.add_argument("--encode", dest="encode", default=0, type=int, help="encode is 0 or 1, default 0")
parser.add_argument("--ntrials", dest="ntrials", default=20, type=int, help="Number of trials to run for hyperparameter tuning")
# model-specific arguments
# (non-tunable)
parser.add_argument("--model", dest="model", default="baseline_lstm", help="Name of model to use")
parser.add_argument("--epochs", dest="epochs", type=int, default=10, help="Number of epochs to train for")
parser.add_argument("--patience", dest="patience", type=int, default=10, help="Learning rate decay scheduler patience, number of epochs")
# (tunable arguments)
parser.opt_list("--batch-size", dest="batch_size", type=int, default=100, help="Size of the minibatch",
tunable=False, options=[32, 64, 128, 256])
parser.opt_range("--learning-rate", dest="learning_rate", type=float, default=1e-3, help="Learning rate for training",
tunable=True, low=1e-3, high=1e-1, nb_samples=4)
parser.opt_list("--hidden-size", dest="hidden_size", type=int, default=100, help="Dimension of hidden layers",
tunable=False, options=[32, 64, 128, 256])
parser.opt_list('--optimizer', dest="optimizer", type=str, default='SGD', help='Optimizer to use (default: SGD)',
tunable=False, options=['SGD', 'Adam'])
parser.opt_range('--weight-decay', dest="weight_decay", type=float, default=1e-5,
help='Weight decay for L2 regularization.',
tunable=True, low=1e-6, high=1e-1, nb_samples=10)
parser.opt_list('--frame-freq', dest="frame_freq", type=int, default=5,
help='Frequency for sub-sampling frames from a video',
tunable=True, options=[10, 30, 60, 75, 100])
# (tcn-only arguments)
parser.opt_list('--dropout', dest="dropout", type=float, default=0.05, help='Dropout applied to layers (default: 0.05)',
tunable=True, options=[0.05, 0.1, 0.3, 0.5, 0.7])
parser.opt_list('--levels', dest="levels", type=int, default=8, help='# of levels for TCN (default: 8)',
tunable=True, options=[6, 8, 10, 12])
# LSTM only arguments
parser.opt_list('--num_layers', dest="num_layers", type=int, default=1, help='# of layers in LSTM (default:1',
tunable=True, options=[1, 2, 3, 4, 5])
# program arguments (dataset and logger paths)
parser.add_argument("--raw_data_path", dest="raw_data_path", default="/mnt/disks/disk1/raw", help="Path to raw dataset")
parser.add_argument('--proc_data_path', dest="proc_data_path", default="/mnt/disks/disk1/processed", help="Path to processed dataset")
parser.add_argument("--log", dest="log", default='', help="Unique log directory name under log/. If the name is empty, do not store logs")
parser.add_argument("--checkpoint", dest="checkpoint", type=str, default="", help="Path to the .pth checkpoint file. Used to continue training from checkpoint")
# create argparser
args = parser.parse_args()
return args
def main():
parent_parser = HyperOptArgumentParser(strategy="grid_search", add_help=False)
logdir = "logs"
parent_parser.add_argument(
"--test_tube_save_path", default=os.path.join(logdir, "test_tube_data")
)
parent_parser.add_argument(
"--model_save_path", default=os.path.join(logdir, "model_weights")
)
parent_parser.add_argument(
"--experiment_name", default=os.path.join(logdir, "vampire")
)
parser = VAMPIRE.add_model_specific_args(parent_parser, ".")
hparams = parser.parse_args()
model = VAMPIRE(hparams)
exp = Experiment(
name=hparams.experiment_name,
save_dir=hparams.test_tube_save_path,
autosave=False,
)
exp.argparse(hparams)
exp.save()
trainer = Trainer(experiment=exp, fast_dev_run=False)
trainer.fit(model)
def get_args():
parser = HyperOptArgumentParser()
parser.add_argument('--save-path',
metavar='DIR',
default="./work_dir",
type=str,
help='path to save output')
parser.add_argument('--gpus', type=str, default='0', help='how many gpus')
parser.add_argument('--dist_backend',
type=str,
default='dp',
choices=('dp', 'ddp', 'ddp2'),
help='supports three options dp, ddp, ddp2')
parser.add_argument('--use-16bit',
dest='use_16bit',
action='store_true',
help='if true uses 16 bit precision')
parser.add_argument('--eval',
'--evaluate',
dest='evaluate',
action='store_true',
help='evaluate model on validation set')
parser.add_argument('--seed', default=8465, type=int)
parser = CGCNModel.add_model_specific_args(parser)
return parser.parse_args()
def add_model_specific_args(parent_parser, root_dir): # pragma: no cover
"""
Parameters you define here will be available to your model through self.hparams
:param parent_parser:
:param root_dir:
:return:
"""
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
# param overwrites
# parser.set_defaults(gradient_clip_val=5.0)
# network params
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
parser.add_argument('--in_features', default=28 * 28, type=int)
parser.add_argument('--out_features', default=10, type=int)
# use 500 for CPU, 50000 for GPU to see speed difference
parser.add_argument('--hidden_dim', default=50000, type=int)
# data
parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
# training params (opt)
parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
options=[0.0001, 0.0005, 0.001, 0.005],
tunable=False)
parser.opt_list('--optimizer_name', default='adam', type=str,
options=['adam'], tunable=False)
# if using 2 nodes with 4 gpus each the batch size here
# (256) will be 256 / (2*8) = 16 per gpu
parser.opt_list('--batch_size', default=256 * 8, type=int,
options=[32, 64, 128, 256], tunable=False,
help='batch size will be divided over all GPUs being used across all nodes')
return parser
def add_model_specific_args(parent_parser):
p = HyperOptArgumentParser(parents=parent_parser, add_help=False, conflict_handler='resolve')
p.add_argument('--dense_encoder', default=False, type=bool)
p.add_argument('--code_size', default=512, type=int)
p.add_argument('--out_channels', default=3, type=int)
p.add_argument('--decoder_convl', default=5, type=int)
if not parent_parser: # Name clash with parent
p.add_argument('--in_channels', default=3, type=int)
return p
def get_default_parser(strategy, root_dir):
possible_model_names = list(AVAILABLE_MODELS.keys())
parser = HyperOptArgumentParser(strategy=strategy, add_help=False)
add_default_args(parser,
root_dir,
possible_model_names=possible_model_names,
rand_seed=SEED)
return parser
def hparams(self):
parser = HyperOptArgumentParser()
# metric mode and monitor are hparams required by COMET models
# and lightning trainer.
parser.add_argument("--monitor", default="slot_error_rate")
parser.add_argument("--metric_mode", default="min")
parser = TransformerTagger.add_model_specific_args(parser)
hparams, _ = parser.parse_known_args([])
return hparams
def add_model_specific_args(parent: HyperOptArgumentParser, root_dir):
parser = HyperOptArgumentParser(strategy=parent.strategy, parents=[parent])
parser.add_argument(
"--data_root",
default=os.path.join(root_dir, "livedoor-news-corpus", "text"),
)
parser.add_argument("--max_vocab", default=30000, type=int)
parser.add_argument("--latent_dim", default=80, type=int)
parser.add_argument("--encoder_num_layers", default=2, type=int)
return parser
def add_model_specific_args(parent_parser):
# nfeat should be inferred from data later
parser = HyperOptArgumentParser(strategy=parent_parser.strategy,
parents=[parent_parser],
add_help=False)
# model
parser.add_argument('--nhid', type=int, default=32)
parser.add_argument('--nlayer', type=int, default=2)
parser.add_argument('--dropout', type=float, default=0)
# optimizer
parser.add_argument('--learning_rate', type=float, default=0.001)
parser.add_argument('--weight_decay', type=float, default=5e-4)
return parser
def add_model_specific_args(parent_parser, root_dir):
"""
Specify the hyperparams for this LightningModule
"""
# MODEL specific
parser = HyperOptArgumentParser(strategy='grid_search',
parents=[parent_parser])
parser.add_argument('--learning_rate', default=0.01, type=float)
parser.add_argument('--batch_size', default=128, type=int)
# parser.add_argument('--split_seed', default=123, type=int)
parser.opt_list('--split_seed',
default=1,
type=int,
tunable=False,
options=[1])
# options=list(1000+np.arange(20).astype(np.int32)))
parser.add_argument('--data_root',
default=Path(root_dir) / 'data',
type=Path)
parser.add_argument('--monitor_preds',
action='store_true',
default=False,
help='export histograms of preds')
parser.add_argument('--trn_subsample_pct',
default=1.0,
type=float,
help='subsample percentage of training data')
parser.opt_list(
'--trn_nb',
default=0,
type=int,
help='number of training samples (0 = take all samples)',
tunable=True,
options=[2000, 1500, 1000, 500])
# options=[750, 500])
# parser.add_argument('--trn_nb', default=0, type=int, help='number of training samples (0 = take all samples)')
parser.opt_list('--outcome',
default='malignancy',
type=str,
tunable=True,
options=[
'sublety', 'internalstructure', 'calcification',
'sphericity', 'margin', 'lobulation',
'spiculation', 'texture', 'malignancy'
])
# training specific (for this model)
parser.add_argument('--max_nb_epochs', default=250, type=int)
return parser
def test_preprocessing_noise(self):
dataset = MaestroDataset('/Volumes/Elements/Datasets/maestro-v2.0.0',
item_length=176400,
sampling_rate=44100,
mode='validation',
max_file_count=20,
shuffle_with_seed=123)
parser = HyperOptArgumentParser(strategy='random_search',
add_help=False)
#parser = ArgumentParser()
parser = ContrastivePredictiveSystem.add_model_specific_args(
parser, root_dir='../')
hparams = parser.parse_args()
preprocessing = PreprocessingModule(hparams)
pass
def main():
parser = HyperOptArgumentParser(
description='Train a PyTorch Lightning model on Yest dataset',
strategy='random_search'
)
parser.opt_list('--nb_layers', default=2, type=int, tunable=False, options=[2, 4, 8])
parser.opt_range('--layer_size', default=20, type=int, tunable=False, low=10, high=200, nb_samples=10, help="size of the hidden layer")
parser.add_argument('--model', default="model.ptl", help="path to save the model")
parser.add_argument('--train', default="yeast_train.svm", help="path to the training data")
parser.add_argument('--val', default="yeast_test.svm", help="path to the training data")
hparams = parser.parse_args()
hparams.optimize_parallel_cpu(train_main, nb_trials=20, nb_workers=8)
def add_model_specific_args(parent_parser):
parser = HyperOptArgumentParser(strategy=parent_parser.strategy,
parents=[parent_parser])
# dataset params
parser.add_argument('--data_dir', default=None)
parser.add_argument('--word_vec_dir', default=None)
parser.add_argument('--dataset', default=None)
parser.add_argument('--n_sample', default=-1, type=float)
parser.add_argument('--n_aug', default=-1, type=int)
parser.add_argument('--pwe', default=0,
type=int) # pretrained word embedding
# model params
parser.add_argument('--embed_dim', default=200, type=int)
parser.opt_list('--hidden_dim',
default=64,
type=int,
options=[64, 128, 256],
tunable=True)
parser.add_argument('--n_layers', default=1, type=int)
parser.opt_list('--dropout',
default=0.1,
type=float,
options=[0.1, 0.2, 0.3, 0.5],
tunable=True)
# training params
parser.add_argument('--max_epochs', default=5, type=int)
parser.opt_list('--learning_rate',
default=4.0,
type=float,
options=[0.05, 0.5, 5],
tunable=True)
parser.opt_list('--scheduler_gamma',
default=0.9,
type=float,
options=[0.9, 0.5, 0.1],
tunable=True)
parser.opt_list('--batch_size',
default=32,
type=int,
options=[16, 32, 64],
tunable=True)
parser.add_argument('--seed', default=-1, type=int)
return parser
def add_model_specific_args(parent_parser, root_dir):
"""
Parameters you define here will be available to your model through self.hparams
:param parent_parser:
:param root_dir:
:return:
"""
parser = HyperOptArgumentParser(strategy=parent_parser.strategy,
parents=[parent_parser])
# param overwrites
# parser.set_defaults(gradient_clip=5.0)
# network params
parser.opt_list('--drop_prob',
default=0.2,
options=[0.2, 0.5],
type=float,
tunable=False)
parser.add_argument('--in_features', default=28 * 28)
parser.add_argument('--out_features', default=10)
parser.add_argument(
'--hidden_dim', default=50000
) # use 500 for CPU, 50000 for GPU to see speed difference
# data
parser.add_argument('--data_root',
default=os.path.join(root_dir, 'mnist'),
type=str)
# training params (opt)
parser.opt_list('--learning_rate',
default=0.001,
type=float,
options=[0.0001, 0.0005, 0.001, 0.005],
tunable=False)
parser.opt_list('--batch_size',
default=256,
type=int,
options=[32, 64, 128, 256],
tunable=False)
parser.opt_list('--optimizer_name',
default='adam',
type=str,
options=['adam'],
tunable=False)
return parser
def test_add_to_parser():
parser = HyperOptArgumentParser(strategy='grid_search')
utils.add_to_parser(parser, 'test0', '4')
utils.add_to_parser(parser, 'test1', '5')
utils.add_to_parser(parser, 'test2', [1, 2, 3])
namespace, _ = parser.parse_known_args(['--test0', '3'])
# single argument
assert namespace.test0 == '3' # user defined arg
assert namespace.test1 == '5' # default arg
# list argument
assert namespace.test2 is None
assert parser.opt_args['--test2'].opt_values == [1, 2, 3]
assert parser.opt_args['--test2'].tunable
def add_model_specific_args(parent_parser):
parser = HyperOptArgumentParser(parents=[parent_parser],
add_help=False)
parser.opt_list(
'--hidden_dim',
type=int,
default=256,
options=[256, 128],
tunable=False,
help=
'itermediate layers dimension before embedding for default encoder/decoder'
)
parser.opt_list('--latent_dim',
type=int,
default=128,
options=[64, 128],
tunable=True,
help='dimension of latent variables z')
parser.opt_list('--learning_rate',
type=float,
default=0.001,
options=[1e-3, 1e-4, 1e-5],
tunable=True)
parser.opt_list('--batch_size',
type=int,
default=16,
options=[16],
tunable=False)
# fixed parameters
parser.add_argument('--input_width',
type=int,
default=306 * 6,
help='input image width')
parser.add_argument('--input_height', type=int, default=256)
parser.add_argument('--output_width', type=int, default=306)
parser.add_argument('--output_height', type=int, default=256)
parser.add_argument('--in_channels', type=int, default=3)
parser.add_argument('--link',
type=str,
default='/scratch/ab8690/DLSP20Dataset/data')
#parser.add_argument('--link', type=str, default='/Users/annika/Developer/driving-dirty/data')
parser.add_argument('--output_img_freq', type=int, default=500)
return parser
def run_train():
# dirs
root_dir = 'saved_seed2'
demo_log_dir = os.path.join(root_dir, 'rnn')
checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
# although we user hyperOptParser, we are using it only as argparse right now
parent_parser = HyperOptArgumentParser(strategy='grid_search',
add_help=False)
# gpu args
parent_parser.add_argument('--gpus',
type=str,
default='-1',
help='how many gpus to use in the node.'
'value -1 uses all the gpus on the node')
parent_parser.add_argument('--test_tube_save_path',
type=str,
default=test_tube_dir,
help='where to save logs')
parent_parser.add_argument('--model_save_path',
type=str,
default=checkpoint_dir,
help='where to save model')
parent_parser.add_argument('--experiment_name',
type=str,
default='inner',
help='test tube exp name')
parent_parser.add_argument('--dataset_prefix',
type=str,
default='nash',
help='[ad, idf, nash]')
# allow model to overwrite or extend args
parser = MyModel.add_model_specific_args(parent_parser, root_dir)
hyperparams = parser.parse_args()
hyperparams.experiment_name = hyperparams.dataset_prefix
# ---------------------
# RUN TRAINING
# ---------------------
# run on HPC cluster
print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
main(hyperparams)
def add_model_specific_args(parent_parser):
parser = HyperOptArgumentParser(parents=[parent_parser],
add_help=False)
# want to optimize this parameter
#parser.opt_list('--batch_size', type=int, default=16, options=[16, 10, 8], tunable=False)
parser.opt_list('--learning_rate',
type=float,
default=0.001,
options=[1e-3, 1e-4, 1e-5],
tunable=True)
parser.add_argument('--batch_size', type=int, default=10)
# fixed arguments
parser.add_argument('--output_img_freq', type=int, default=100)
parser.add_argument('--unfreeze_epoch_no', type=int, default=0)
parser.add_argument('--mse_loss', default=False, action='store_true')
return parser