from funkyyak import grad, kylist
from hypergrad.data import load_data_dicts
from hypergrad.nn_utils import make_nn_funs, VectorParser, logit, inv_logit, fill_parser
from hypergrad.optimizers import simple_sgd, sgd_parsed
from hypergrad.util import RandomState
import whetlab
parameters = {
'init_log_alphas': {
'min': -3,
'max': 2,
'type': 'float'
},
'init_invlogit_betas': {
'min': inv_logit(0.1),
'max': inv_logit(0.999),
'type': 'float'
},
'init_log_param_scale': {
'min': -5,
'max': 1,
'type': 'float'
}
}
outcome = {'name': 'Training Gain'}
scientist = whetlab.Experiment(
name="ICML Hypergrad paper - optimize initial values - v2 no randomness",
description="Vanilla tuning of hyperparameters. v2",
parameters=parameters,
outcome=outcome)
from hypergrad.optimizers import sgd4, rms_prop, adam
# ----- Fixed params -----
layer_sizes = [784, 10]
batch_size = 200
N_iters = 60
N_classes = 10
N_train = 1000
N_valid = 10**3
N_tests = 10**3
N_batches = N_train / batch_size
#N_iters = N_epochs * N_batches
# ----- Initial values of learned hyper-parameters -----
init_log_L2_reg = 0.0
init_log_alphas = -2.0
init_invlogit_betas = inv_logit(0.9)
init_log_param_scale = 0.0
# ----- Superparameters -----
meta_alpha = 0.04
N_meta_iter = 100
global_seed = npr.RandomState(3).randint(1000)
def fill_parser(parser, items):
partial_vects = [np.full(parser[name].size, items[i])
for i, name in enumerate(parser.names)]
return np.concatenate(partial_vects, axis=0)
def run():
train_data, valid_data, tests_data = load_data_dicts(N_train, N_valid, N_tests)
parser, pred_fun, loss_fun, frac_err = make_nn_funs(layer_sizes)
# ----- Fixed params -----
layer_sizes = [784, 100, 10]
batch_size = 200
N_iters = 100
N_classes = 10
N_train = 1000
N_valid = 10000
N_tests = 10000
N_learning_checkpoint = 2
thin = np.ceil(N_iters / N_learning_checkpoint)
# ----- Initial values of learned hyper-parameters -----
init_log_L2_reg = 0.0
init_log_L2_reg_noise = 0.0
init_log_alphas = -1.0
init_invlogit_betas = inv_logit(0.5)
init_log_param_scale = -3.0
# ----- Superparameters -----
meta_alpha = 0.4
N_meta_iter = 50
seed = 0
def fill_parser(parser, items):
partial_vects = [
np.full(parser[name].size, items[i])
for i, name in enumerate(parser.names)
]
return np.concatenate(partial_vects, axis=0)
def test_inv_logit():
assert np.allclose(inv_logit(logit(0.5)), 0.5, rtol=1e-3, atol=1e-4)
assert np.allclose(inv_logit(logit(0.6)), 0.6, rtol=1e-3, atol=1e-4)
assert np.allclose(inv_logit(logit(0.1)), 0.1, rtol=1e-3, atol=1e-4)
assert np.allclose(inv_logit(logit(0.2)), 0.2, rtol=1e-3, atol=1e-4)
"""Find good initial values using Whetlab."""
import numpy as np
import pickle
from collections import defaultdict
from funkyyak import grad, kylist
from hypergrad.data import load_data_dicts
from hypergrad.nn_utils import make_nn_funs, VectorParser, logit, inv_logit, fill_parser
from hypergrad.optimizers import simple_sgd, sgd_parsed
from hypergrad.util import RandomState
import whetlab
parameters = { 'init_log_alphas':{'min':-3, 'max':2, 'type':'float'},
'init_invlogit_betas':{'min':inv_logit(0.1), 'max':inv_logit(0.999),'type':'float'},
'init_log_param_scale':{'min':-5, 'max':1,'type':'float'}}
outcome = {'name':'Training Gain'}
scientist = whetlab.Experiment(name="ICML Hypergrad paper - optimize initial values",
description="Vanilla tuning of hyperparameters.",
parameters=parameters,
outcome=outcome)
# ----- Fixed params -----
layer_sizes = [784, 50, 50, 50, 10]
batch_size = 200
N_iters = 100
N_classes = 10
N_train = 10000
N_valid = 10000
N_tests = 10000
N_learning_checkpoint = 10
"""Find good initial values using Whetlab."""
import pickle
from collections import defaultdict
import numpy as np
import whetlab
from funkyyak import grad, kylist
from hypergrad.data import load_data_dicts
from hypergrad.nn_utils import make_nn_funs, VectorParser, logit, inv_logit, fill_parser
from hypergrad.optimizers import sgd_parsed
from hypergrad.util import RandomState
parameters = {
"init_log_alphas": {"min": -3, "max": 2, "type": "float"},
"init_invlogit_betas": {"min": inv_logit(0.1), "max": inv_logit(0.999), "type": "float"},
"init_log_param_scale": {"min": -5, "max": 1, "type": "float"},
}
outcome = {"name": "Training Gain"}
scientist = whetlab.Experiment(
name="ICML Hypergrad paper - optimize initial values",
description="Vanilla tuning of hyperparameters.",
parameters=parameters,
outcome=outcome,
)
# ----- Fixed params -----
layer_sizes = [784, 50, 50, 50, 10]
batch_size = 200
N_iters = 100
N_classes = 10
def test_inv_logit():
assert np.allclose(inv_logit(logit(0.5)), 0.5, rtol=1e-3, atol=1e-4)
assert np.allclose(inv_logit(logit(0.6)), 0.6, rtol=1e-3, atol=1e-4)
assert np.allclose(inv_logit(logit(0.1)), 0.1, rtol=1e-3, atol=1e-4)
assert np.allclose(inv_logit(logit(0.2)), 0.2, rtol=1e-3, atol=1e-4)