def search_default(cls) -> "FullyConv1Resnet.ModelParams":
return cls(
num_groups=params.Discrete([8, 16, 32, 64]),
num_blocks_per_residual=params.Integer(1, 5),
activation=params.Categorical([nn.LeakyReLU, nn.GELU]),
do_include_first_norm=params.Boolean(p_true=0.8),
)
class ModelHyperParams(params.ParameterSet):
num_hidden_layers = params.Integer(1, 4)
num_neurons_per_layer = params.Discrete(
np.logspace(2, 7, num=6, base=2, dtype=np.int))
dropout_rate = params.Float(0.0, 0.99)
activation = params.Categorical(['relu', 'sigmoid'])
output_dir = '/tmp/output'
filter_size = 3
max_seq_len=2**11,
batch_size=32,
)
# neck_hp.data.dataset_name = 'num=1000_02b7'
hp.data.dataset_name = 'num=4000_9b9f'
hp.data.do_ignore_cached_dataset = False
hp.data.seed = 42
hp.data.num_workers = 0
hp.data.num_gpus = 1
hp.data.num_cpus = 4
hp.opt.search_metric = 'valid_loss_total'
hp.opt.search_mode = 'min'
hp.opt.num_epochs = 100
hp.opt.lr = params.Discrete([1e-4, 1e-3, 1e-2, 1e-1])
hp.opt.min_lr = 1e-6
hp.opt.patience = 16
hp.exp.project_name = 'tablestakes'
hp.exp.experiment_name = 'korv_which'
hp.exp.experiment_tags = [
'korv_which', 'conv', 'sharp', 'search', 'v0.0.1'
]
hp.exp.sources_glob_str = constants.THIS_DIR.parent.parent / '**/*.py'
hp.embed.dim = 16
hp.embed.requires_grad = True
hp.conv.num_features = params.Discrete([32, 64, 128, 256])
hp.conv.num_layers = params.Integer(2, 11)
env['IMAGE_URI'] = CONTAINER_IMAGE_URI
env['BUCKET_NAME'] = GCLOUD_BUCKET_NAME
# Create a Cloud AI Platform Hyperparameter Search object
search = search.HyperparamSearchSpec(
max_trials=10,
max_parallel_trials=5,
max_failed_trials=2,
hyperparameter_metric_tag='val_acc',
)
# Add parameter search ranges for this problem.
my_param_ranges = train.MyParams(
activation=params.Categorical(['relu', 'tanh']),
num_layers=params.Integer(min_value=1, max_value=3),
num_neurons=params.Discrete(np.logspace(2, 8, num=7, base=2)),
dropout_rate=params.Double(min_value=-0.1, max_value=0.9),
learning_rate=params.Discrete(np.logspace(-6, 2, 17, base=10)),
batch_size=params.Integer(min_value=1, max_value=128),
)
search.add_parameters(my_param_ranges)
# Call a bash script to build a docker image for this repo, submit it to the docker registry defined in the script
# and run a training job on the Cloud AI Platform using this container and these hyperparameter ranges.
this_dir = Path(__file__).resolve().parent
retcode = subprocess.call([this_dir / 'build_push.sh'], env=env)
if retcode:
raise ValueError(f"Got returncode: {retcode}")
search.run_from_container(
gcloud_project_name=GCLOUD_PROJECT_NAME,
utils.print_dict(best_trial.last_result)
utils.hprint('best_trial.config', do_include_pre_break_line=True)
utils.print_dict(best_trial.config)
if __name__ == '__main__':
fast_dev_run = False
dp = data_module.DataParams(
# dataset_name='num=100_057b',
dataset_name='num=1000_2cfc', )
hp = ts_model.TotalParams(
data=dp,
max_seq_len=8192,
batch_size=params.Discrete([4, 8, 16, 32, 64, 128, 256]),
)
hp.data.do_ignore_cached_dataset = False
hp.data.seed = 42
hp.data.num_workers = 4
hp.data.num_gpus = 1
hp.data.num_cpus = 4
hp.opt.search_metric = 'valid/loss'
hp.opt.search_mode = 'min'
hp.opt.num_epochs = 4
hp.opt.lr = params.Discrete(
[1e-5, 3e-5, 1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1])
hp.opt.patience = 10