def train_test(train, labels, test, **parameters):
params = {
'name': name_from_file(),
'max_epochs': 600,
'auto_stopping': True,
'adaptive_weight_decay': False,
'save_snapshots_stepsize': None,
'epoch_steps': None,
#'dense3_size': 0,
'momentum_scaling': 1200,
'dense1_nonlinearity': 'rectify',
'dense1_init': 'glorot_uniform',
'dense2_nonlinearity': 'rectify',
'dense2_init': 'glorot_uniform',
'batch_size': 128,
'learning_rate': 0.0003,
'learning_rate_scaling': 1000,
'momentum': 0.98,
'dense1_size': 700,
'dense2_size': 550,
'dense3_size': 400,
#'nn__dense3_size': randint(low = 100, high = 400),
'dropout0_rate': 0.,
'dropout1_rate': 0.1,
'dropout2_rate': 0.45,
'dropout3_rate': 0.58,
#'nn__dropout3_rate': triang(loc = 0, c = 0, scale = 1),
#'nn__weight_decay': norm(0.00006, 0.0001),
}
params.update(parameters)
estimator = Pipeline([
('row', PositiveSparseRowFeatureGenerator()),
('gen23',
PositiveSparseFeatureGenerator(difficult_classes=(2, 3),
extra_features=40)),
('gen234',
PositiveSparseFeatureGenerator(difficult_classes=(2, 3, 4),
extra_features=40)),
('gen19',
PositiveSparseFeatureGenerator(difficult_classes=(1, 9),
extra_features=63)),
('log',
LogTransform()), # log should be after integer feats but before dist
#('distp31', DistanceFeatureGenerator(n_neighbors = 3, distance_p = 1)),
#('distp52', DistanceFeatureGenerator(n_neighbors = 5, distance_p = 2)),
('scale03', MinMaxScaler(feature_range=(0, 3))
), # scale should apply to int and float feats
('nn', NNet(**params)),
])
estimator.fit(train, labels)
prediction = estimator.predict_proba(test)
return prediction
def optimize_map(index):
random = Random(index + SEED)
name = 'many_{1:d}'.format(name_from_file(), index)
if isfile(join(NNET_STATE_DIR, name + '_best.net.npz')):
print name, 'best found'
pretrain = join(NNET_STATE_DIR, name + '_best.net.npz')
elif isfile(join(NNET_STATE_DIR, name + '.net.npz')):
print name, 'found'
pretrain = join(NNET_STATE_DIR, name + '.net.npz')
else:
print name, 'NOT found'
pretrain = None
#return
params = {
'name':
name,
'dense1_size':
64 + random.randint(0, 384),
'dense2_size':
64 + random.randint(0, 384),
'dense3_size':
64 + random.randint(0, 128),
'learning_rate':
0.001 + 0.01 * random.random(),
'learning_rate_scaling':
500,
'momentum':
0.99,
'momentum_scaling':
10,
'dropout1_rate':
0.5,
'dropout2_rate':
0.5,
'weight_decay':
0.00025 * (random.random() + random.random() + random.random()),
'max_epochs':
1500,
'extra_feature_count':
163,
'pretrain':
pretrain,
'save_snapshots_stepsize':
1500,
}
if index < 10:
print index, params
return
return optimize_NN(debug=True, test_only=True, **params)
def optimize_map(index):
random = Random(index + SEED)
params = {
'name':
'{0:s}_{1:d}'.format(name_from_file(), index),
'dense1_size':
64 + random.randint(0, 384),
'dense2_size':
64 + random.randint(0, 384),
'dense3_size':
64 + random.randint(0, 128),
'learning_rate':
0.001 + 0.01 * random.random(),
'learning_rate_scaling':
500,
'momentum':
0.99,
'momentum_scaling':
10,
'dropout1_rate':
0.5,
'dropout2_rate':
0.5,
'weight_decay':
0.00025 * (random.random() + random.random() + random.random()),
'max_epochs':
1500,
'extra_feature_count':
163,
'pretrain':
False,
'save_snapshots_stepsize':
1500,
}
with open(join(NNET_STATE_DIR, 'param_{0:03d}.json'.format(index)),
'w+') as fh:
print 'writing params for #{0:d}'.format(index)
dump(obj=params, fp=fh)
return optimize_NN(debug=True, **params)
'dense2_size': 0,
'dense3_size': None,
'learning_rate': 0.001,
'learning_rate_scaling': 100,
'momentum': 0.9,
'momentum_scaling': 10,
'dropout1_rate': 0.5,
'dropout2_rate': None,
'dropout3_rate': None,
'weight_decay': 0,
'max_epochs': 1500,
'auto_stopping': False,
'extra_feature_count': 0,
'pretrain': False, # keep this OFF
'save_snapshots_stepsize': 500,
'name': name_from_file(),
'outlier_method': 'EE',
'outlier_frac': None,
'normalize_log': True,
'use_calibration': False,
'use_rescale_priors': True,
'extra_feature_seed': 0,
'test_data_confidence': None,
}
prediction = train_test_NN(train, train_labels, test, **params)
# do things with prediction
print calc_logloss(prediction, test_labels)
save('nnpred.npy', prediction)
train_data, true_labels = get_training_data()[:2]
validator = SampleCrossValidator(train_data,
true_labels,
rounds=1,
test_frac=0.2,
use_data_frac=1) # 0.3!!
optimizer = ParallelGridOptimizer(train_test_func=train_test,
validator=validator,
use_caching=False,
process_count=max(cpu_count() - 1, 1),
**{
'weight_decay':
logspace(-1, -7, base=10, num=30),
}).readygo(
save_fig_basename=name_from_file(),
log_name=name_from_file() + '_stats.txt')
"""
1 0.5254 4.89390091848e-05
2 0.5257 3.03919538231e-05
3 0.5284 1.88739182214e-05
4 0.5308 7.88046281567e-05
5 0.5313 1.17210229753e-05
6 0.5315 7.27895384398e-06
7 0.5317 4.52035365636e-06
8 0.5320 2.80721620394e-06
9 0.5336 1.08263673387e-06
10 0.5338 1e-07
11 0.5339 4.17531893656e-07
12 0.5339 1.7433288222e-06
13 0.5340 1.61026202756e-07
'dense1_size': 400,
'dense2_size': 300,
'dense3_size': 300,
'dense1_nonlinearity': 'rectify',
'dense1_init':
'glorot_uniform', # uniform to reduce randomness
'momentum': 0.98,
'momentum_scaling': 10,
'learning_rate_scaling': 500,
'batch_size': 128,
'dropout0_rate': 0,
})),
]),
param_distributions={
'nn__name': [
'final_{0:s}_{1:03d}'.format(name_from_file(), k)
for k in range(10000)
],
'nn__learning_rate':
norm(0.0001, 0.00003),
'nn__dropout1_rate':
uniform(loc=0.00, scale=0.25),
'nn__dropout2_rate':
uniform(loc=0.30, scale=0.60 - 0.30),
'nn__dropout3_rate':
uniform(loc=0.40, scale=0.80 - 0.40),
'nn__weight_decay':
[0.01, 0.007, 0.0025, 0.001, 0.0007, 0.00025, 0.0001, 0.00007],
},
fit_params={
'nn__random_sleep': 600,
from nnet.oldstyle.base_optimize import name_from_file
from nnet.scikit import NNet
from utils.loading import get_training_data
train, labels = get_training_data()[:2]
#test = get_testing_data()[0]
#train, labels = expand_from_test(train, labels, test, confidence = 0.9)
#gen = PositiveSparseRowFeatureGenerator()
#train = gen.fit_transform(train, labels)
#test = gen.transform(test, labels)
#random = RandomState()
opt = RandomizedSearchCV(
estimator=NNet(
name=name_from_file(),
auto_stopping=True,
max_epochs=1500, # binom(n = 4000, p = 0.25)
),
param_distributions={
'dense1_size': randint(low=100, high=1200),
'dense2_size': randint(low=50, high=900),
'dense3_size': randint(low=25, high=700),
},
fit_params={},
n_iter=600,
n_jobs=cpus - 1,
scoring=log_loss_scorer,
refit=False,
pre_dispatch=3,
cv=ShuffleSplit(
'nn__dense1_size': randint(low=400, high=900),
'nn__dense2_size': randint(low=300, high=700),
'nn__dense3_size': randint(low=200, high=500),
'nn__dropout0_rate': triang(loc=0, c=0, scale=0.5),
'nn__dropout1_rate': uniform(loc=0, scale=0.8),
'nn__dropout2_rate': uniform(loc=0, scale=0.8),
'nn__dropout3_rate': uniform(loc=0, scale=0.8),
#'nn__weight_decay': norm(0.00006, 0.0001),
},
fit_params={
'nn__random_sleep': 600,
},
n_iter=30,
n_jobs=cpus,
scoring=get_logloss_loggingscorer(join(
OPTIMIZE_RESULTS_DIR, '{0:s}.log'.format(name_from_file())),
treshold=.7),
iid=False,
refit=True,
pre_dispatch=cpus + 2,
cv=ShuffleSplit(
n=train.shape[0],
n_iter=1,
test_size=0.2,
random_state=random,
),
random_state=random,
verbose=bool(VERBOSITY),
error_score=-1000000,
)
'epoch_steps': None,
'dense1_size': 70,
'dense2_size': 55,
'dense3_size': 0, #todo
'dense1_nonlinearity': 'rectify',
'dense1_init':
'glorot_uniform', # uniform to reduce randomness
'momentum': 0.98,
'momentum_scaling': 1,
'learning_rate_scaling': 100,
'batch_size': 128,
})),
]),
param_distributions={
'nn__name':
['nn_{0:s}_{1:03d}'.format(name_from_file(), k) for k in range(10000)],
'nn__learning_rate':
norm(0.0005, 0.0003),
'nn__dropout0_rate':
triang(loc=0, c=0, scale=0.5),
'nn__dropout1_rate':
uniform(loc=0.00, scale=0.50),
'nn__dropout2_rate':
uniform(loc=0.15, scale=0.70 - 0.15),
#'nn__dropout3_rate': uniform(loc = 0.30, scale = 0.80-0.30), #todo
#'nn__weight_decay': norm(0.00006, 0.0001),
},
fit_params={
'nn__random_sleep': 80,
},
n_iter=40,
