def optimize_map(index):
random = Random(index + SEED)
name = 'many_{1:d}'.format(name_from_file(), index)
if isfile(join(NNET_STATE_DIR, name + '_best.net.npz')):
print name, 'best found'
pretrain = join(NNET_STATE_DIR, name + '_best.net.npz')
elif isfile(join(NNET_STATE_DIR, name + '.net.npz')):
print name, 'found'
pretrain = join(NNET_STATE_DIR, name + '.net.npz')
else:
print name, 'NOT found'
pretrain = None
#return
params = {
'name':
name,
'dense1_size':
64 + random.randint(0, 384),
'dense2_size':
64 + random.randint(0, 384),
'dense3_size':
64 + random.randint(0, 128),
'learning_rate':
0.001 + 0.01 * random.random(),
'learning_rate_scaling':
500,
'momentum':
0.99,
'momentum_scaling':
10,
'dropout1_rate':
0.5,
'dropout2_rate':
0.5,
'weight_decay':
0.00025 * (random.random() + random.random() + random.random()),
'max_epochs':
1500,
'extra_feature_count':
163,
'pretrain':
pretrain,
'save_snapshots_stepsize':
1500,
}
if index < 10:
print index, params
return
return optimize_NN(debug=True, test_only=True, **params)
def optimize_map(index):
random = Random(index + SEED)
params = {
'name':
'{0:s}_{1:d}'.format(name_from_file(), index),
'dense1_size':
64 + random.randint(0, 384),
'dense2_size':
64 + random.randint(0, 384),
'dense3_size':
64 + random.randint(0, 128),
'learning_rate':
0.001 + 0.01 * random.random(),
'learning_rate_scaling':
500,
'momentum':
0.99,
'momentum_scaling':
10,
'dropout1_rate':
0.5,
'dropout2_rate':
0.5,
'weight_decay':
0.00025 * (random.random() + random.random() + random.random()),
'max_epochs':
1500,
'extra_feature_count':
163,
'pretrain':
False,
'save_snapshots_stepsize':
1500,
}
with open(join(NNET_STATE_DIR, 'param_{0:03d}.json'.format(index)),
'w+') as fh:
print 'writing params for #{0:d}'.format(index)
dump(obj=params, fp=fh)
return optimize_NN(debug=True, **params)
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'dense1_size': 256,
'dense2_size': 128,
'learning_rate': [0.01, 0.003, 0.001, 0.0003, 0.0001],
'momentum': 0.9,
'dropout1_rate': 0.5,
'rounds': 3,
})
optimize_NN(
debug=False,
**{
'dense1_nonlinearity':
'rectify', # tanh, sigmoid, rectify, leaky2, leaky20, softmax
'dense1_init':
'glorot_normal', # orthogonal, sparse, glorot_normal, glorot_uniform, he_normal, he_uniform
'dense1_size': 256, # hidden neurons in layer (30~1000)
'dense2_size': 0,
'dense3_size': None,
'learning_rate':
0.001, # initial learning rate (learning rate is effectively higher for higher momentum)
'learning_rate_scaling':
100, # progression over time; 0.1 scaled by 10 is 0.01
'momentum': 0.9, # initial momentum
'momentum_scaling': 10, # 0.9 scaled by 10 is 0.99
'dropout1_rate': 0.5, # [0, 0.5]
'dropout2_rate': None, # inherit dropout1_rate if dense2 exists
'dropout3_rate': None, # inherit dropout2_rate if dense3 exist
'weight_decay': 0.00006, # constrain the weights to avoid overfitting
'max_epochs':
500, # it terminates when overfitting or increasing, so just leave high
'auto_stopping':
False, # stop training automatically if it seems to be failing
'extra_feature_count': 0,
'pretrain': 'results/pretrain/single_pretrain_256_0_0.net.npz',
'save_snapshots_stepsize': 500,
'rounds': 3,
})
from nnet.oldstyle.base_optimize import optimize_NN
from nnet.oldstyle.train_test import train_test_NN
def train_test_NN_scale_lr(train, labels, test, learning_rate = None, momentum = None, **kwargs):
print 'rescaling learning rate: {0:.6f} -> {1:.6f} (for momentum {2:.6f})'.format(learning_rate, learning_rate / (1 - momentum), momentum)
return train_test_NN(train, labels, test, learning_rate = learning_rate * (1 - momentum), momentum = momentum, **kwargs)
optimize_NN(debug = False, train_test_func = train_test_NN_scale_lr, **{
'dense1_size': 256,
'dense2_size': 128,
'learning_rate': 0.01,
'learning_rate_scaling': 1000,
'momentum': [0.1, 0.9, 0.99, 0.999],
'momentum_scaling': 100,
'dropout1_rate': 0.5,
'rounds': 3,
})
from nnet.oldstyle.base_optimize import optimize_NN
# previously called 'big'
optimize_NN(debug=False,
**{
'dense1_size': 512,
'dense2_size': 512,
'dense3_size': 256,
'dropout1_rate': [0, 0.5],
'weight_decay': [.001, .0001, .00001, 0],
'max_epochs': 2000,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'dense1_size': 256,
'dense2_size': 128,
'dropout1_rate': 00.5,
'test_data_confidence': [None, 0.98, 0.95, 0.9, 0.8, 0.6],
'rounds': 5,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'dense1_size':
128,
'dense2_size':
64,
'dense1_nonlinearity':
['tanh', 'sigmoid', 'rectify', 'leaky2', 'leaky20', 'softmax'],
'rounds':
3,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False, **{
'normalize_log': [False, True],
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'dense1_size': 128,
'dense2_size': None,
'dense3_size': None,
'learning_rate': 0.01,
'dropout1_rate': 0,
'max_epochs': 3,
'extra_feature_count': 20,
'pretrain': None,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug = False, **{
'dense1_size': 128,
'dense2_size': 64,
'dense1_init': ['orthogonal', 'sparse', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform'],
'rounds': 3,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'dense1_size': 256,
'dense2_size': 256,
'dense3_size': None,
'dropout1_rate': 0.5,
'extra_feature_count': [7, 17, 37, 67, 107, 157, 217, 287],
'rounds': 3,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'dense1_size': 256,
'dense2_size': [64, 128, 256],
'dense3_size': [None, 64],
'learning_rate': [0.015, 0.008, 0.004],
'learning_rate_scaling': 500,
'momentum': 0.99,
'momentum_scaling': 10,
'dropout1_rate': [0, 0.5],
'dropout2_rate': None,
'weight_decay': 0,
'max_epochs': 1500,
'extra_feature_count': 163,
'pretrain': False,
'save_snapshots_stepsize': 1500,
'rounds': 5,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'dense1_size': 256,
'dense2_size': 256,
'dense3_size': 128,
'learning_rate': [0.015, 0.008, 0.004],
'learning_rate_scaling': 500,
'momentum': 0.99,
'momentum_scaling': 10,
'dropout1_rate': 0.5,
'dropout2_rate': 0.5,
'weight_decay':
[0.0005, 0.001, 0.002, 0.003, 0.004, 0.005, 0.01],
'max_epochs': 1500,
'extra_feature_count': 163,
'pretrain': True,
'save_snapshots_stepsize': 1500,
'rounds': 3,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'dense1_nonlinearity': 'rectify',
'dense1_init': 'glorot_normal',
'dense1_size': 256,
'dense2_size': [0, 128],
'dense3_size': None,
'learning_rate': [0.0005, 0.0010, 0.0015],
'learning_rate_scaling': 100,
'momentum': 0.9,
'momentum_scaling': 10,
'dropout1_rate': 0.5,
'dropout2_rate': None,
'dropout3_rate': None,
'weight_decay': [0.00001, 0.00006, 0.00015, 0.00040, 0.00100],
'max_epochs': 500,
'auto_stopping': True,
'extra_feature_count': [0, 163],
'pretrain': None,
'save_snapshots_stepsize': 500,
'rounds': 1,
})
from nnet.oldstyle.base_optimize import optimize_NN
optimize_NN(debug=False,
**{
'use_calibration': [False, True],
'use_rescale_priors': [False, True],
})