def train_test(train, labels, test, **parameters): params = { 'name': name_from_file(), 'max_epochs': 600, 'auto_stopping': True, 'adaptive_weight_decay': False, 'save_snapshots_stepsize': None, 'epoch_steps': None, #'dense3_size': 0, 'momentum_scaling': 1200, 'dense1_nonlinearity': 'rectify', 'dense1_init': 'glorot_uniform', 'dense2_nonlinearity': 'rectify', 'dense2_init': 'glorot_uniform', 'batch_size': 128, 'learning_rate': 0.0003, 'learning_rate_scaling': 1000, 'momentum': 0.98, 'dense1_size': 700, 'dense2_size': 550, 'dense3_size': 400, #'nn__dense3_size': randint(low = 100, high = 400), 'dropout0_rate': 0., 'dropout1_rate': 0.1, 'dropout2_rate': 0.45, 'dropout3_rate': 0.58, #'nn__dropout3_rate': triang(loc = 0, c = 0, scale = 1), #'nn__weight_decay': norm(0.00006, 0.0001), } params.update(parameters) estimator = Pipeline([ ('row', PositiveSparseRowFeatureGenerator()), ('gen23', PositiveSparseFeatureGenerator(difficult_classes=(2, 3), extra_features=40)), ('gen234', PositiveSparseFeatureGenerator(difficult_classes=(2, 3, 4), extra_features=40)), ('gen19', PositiveSparseFeatureGenerator(difficult_classes=(1, 9), extra_features=63)), ('log', LogTransform()), # log should be after integer feats but before dist #('distp31', DistanceFeatureGenerator(n_neighbors = 3, distance_p = 1)), #('distp52', DistanceFeatureGenerator(n_neighbors = 5, distance_p = 2)), ('scale03', MinMaxScaler(feature_range=(0, 3)) ), # scale should apply to int and float feats ('nn', NNet(**params)), ]) estimator.fit(train, labels) prediction = estimator.predict_proba(test) return prediction
def optimize_map(index): random = Random(index + SEED) name = 'many_{1:d}'.format(name_from_file(), index) if isfile(join(NNET_STATE_DIR, name + '_best.net.npz')): print name, 'best found' pretrain = join(NNET_STATE_DIR, name + '_best.net.npz') elif isfile(join(NNET_STATE_DIR, name + '.net.npz')): print name, 'found' pretrain = join(NNET_STATE_DIR, name + '.net.npz') else: print name, 'NOT found' pretrain = None #return params = { 'name': name, 'dense1_size': 64 + random.randint(0, 384), 'dense2_size': 64 + random.randint(0, 384), 'dense3_size': 64 + random.randint(0, 128), 'learning_rate': 0.001 + 0.01 * random.random(), 'learning_rate_scaling': 500, 'momentum': 0.99, 'momentum_scaling': 10, 'dropout1_rate': 0.5, 'dropout2_rate': 0.5, 'weight_decay': 0.00025 * (random.random() + random.random() + random.random()), 'max_epochs': 1500, 'extra_feature_count': 163, 'pretrain': pretrain, 'save_snapshots_stepsize': 1500, } if index < 10: print index, params return return optimize_NN(debug=True, test_only=True, **params)
def optimize_map(index): random = Random(index + SEED) params = { 'name': '{0:s}_{1:d}'.format(name_from_file(), index), 'dense1_size': 64 + random.randint(0, 384), 'dense2_size': 64 + random.randint(0, 384), 'dense3_size': 64 + random.randint(0, 128), 'learning_rate': 0.001 + 0.01 * random.random(), 'learning_rate_scaling': 500, 'momentum': 0.99, 'momentum_scaling': 10, 'dropout1_rate': 0.5, 'dropout2_rate': 0.5, 'weight_decay': 0.00025 * (random.random() + random.random() + random.random()), 'max_epochs': 1500, 'extra_feature_count': 163, 'pretrain': False, 'save_snapshots_stepsize': 1500, } with open(join(NNET_STATE_DIR, 'param_{0:03d}.json'.format(index)), 'w+') as fh: print 'writing params for #{0:d}'.format(index) dump(obj=params, fp=fh) return optimize_NN(debug=True, **params)
'dense2_size': 0, 'dense3_size': None, 'learning_rate': 0.001, 'learning_rate_scaling': 100, 'momentum': 0.9, 'momentum_scaling': 10, 'dropout1_rate': 0.5, 'dropout2_rate': None, 'dropout3_rate': None, 'weight_decay': 0, 'max_epochs': 1500, 'auto_stopping': False, 'extra_feature_count': 0, 'pretrain': False, # keep this OFF 'save_snapshots_stepsize': 500, 'name': name_from_file(), 'outlier_method': 'EE', 'outlier_frac': None, 'normalize_log': True, 'use_calibration': False, 'use_rescale_priors': True, 'extra_feature_seed': 0, 'test_data_confidence': None, } prediction = train_test_NN(train, train_labels, test, **params) # do things with prediction print calc_logloss(prediction, test_labels) save('nnpred.npy', prediction)
train_data, true_labels = get_training_data()[:2] validator = SampleCrossValidator(train_data, true_labels, rounds=1, test_frac=0.2, use_data_frac=1) # 0.3!! optimizer = ParallelGridOptimizer(train_test_func=train_test, validator=validator, use_caching=False, process_count=max(cpu_count() - 1, 1), **{ 'weight_decay': logspace(-1, -7, base=10, num=30), }).readygo( save_fig_basename=name_from_file(), log_name=name_from_file() + '_stats.txt') """ 1 0.5254 4.89390091848e-05 2 0.5257 3.03919538231e-05 3 0.5284 1.88739182214e-05 4 0.5308 7.88046281567e-05 5 0.5313 1.17210229753e-05 6 0.5315 7.27895384398e-06 7 0.5317 4.52035365636e-06 8 0.5320 2.80721620394e-06 9 0.5336 1.08263673387e-06 10 0.5338 1e-07 11 0.5339 4.17531893656e-07 12 0.5339 1.7433288222e-06 13 0.5340 1.61026202756e-07
'dense1_size': 400, 'dense2_size': 300, 'dense3_size': 300, 'dense1_nonlinearity': 'rectify', 'dense1_init': 'glorot_uniform', # uniform to reduce randomness 'momentum': 0.98, 'momentum_scaling': 10, 'learning_rate_scaling': 500, 'batch_size': 128, 'dropout0_rate': 0, })), ]), param_distributions={ 'nn__name': [ 'final_{0:s}_{1:03d}'.format(name_from_file(), k) for k in range(10000) ], 'nn__learning_rate': norm(0.0001, 0.00003), 'nn__dropout1_rate': uniform(loc=0.00, scale=0.25), 'nn__dropout2_rate': uniform(loc=0.30, scale=0.60 - 0.30), 'nn__dropout3_rate': uniform(loc=0.40, scale=0.80 - 0.40), 'nn__weight_decay': [0.01, 0.007, 0.0025, 0.001, 0.0007, 0.00025, 0.0001, 0.00007], }, fit_params={ 'nn__random_sleep': 600,
from nnet.oldstyle.base_optimize import name_from_file from nnet.scikit import NNet from utils.loading import get_training_data train, labels = get_training_data()[:2] #test = get_testing_data()[0] #train, labels = expand_from_test(train, labels, test, confidence = 0.9) #gen = PositiveSparseRowFeatureGenerator() #train = gen.fit_transform(train, labels) #test = gen.transform(test, labels) #random = RandomState() opt = RandomizedSearchCV( estimator=NNet( name=name_from_file(), auto_stopping=True, max_epochs=1500, # binom(n = 4000, p = 0.25) ), param_distributions={ 'dense1_size': randint(low=100, high=1200), 'dense2_size': randint(low=50, high=900), 'dense3_size': randint(low=25, high=700), }, fit_params={}, n_iter=600, n_jobs=cpus - 1, scoring=log_loss_scorer, refit=False, pre_dispatch=3, cv=ShuffleSplit(
'nn__dense1_size': randint(low=400, high=900), 'nn__dense2_size': randint(low=300, high=700), 'nn__dense3_size': randint(low=200, high=500), 'nn__dropout0_rate': triang(loc=0, c=0, scale=0.5), 'nn__dropout1_rate': uniform(loc=0, scale=0.8), 'nn__dropout2_rate': uniform(loc=0, scale=0.8), 'nn__dropout3_rate': uniform(loc=0, scale=0.8), #'nn__weight_decay': norm(0.00006, 0.0001), }, fit_params={ 'nn__random_sleep': 600, }, n_iter=30, n_jobs=cpus, scoring=get_logloss_loggingscorer(join( OPTIMIZE_RESULTS_DIR, '{0:s}.log'.format(name_from_file())), treshold=.7), iid=False, refit=True, pre_dispatch=cpus + 2, cv=ShuffleSplit( n=train.shape[0], n_iter=1, test_size=0.2, random_state=random, ), random_state=random, verbose=bool(VERBOSITY), error_score=-1000000, )
'epoch_steps': None, 'dense1_size': 70, 'dense2_size': 55, 'dense3_size': 0, #todo 'dense1_nonlinearity': 'rectify', 'dense1_init': 'glorot_uniform', # uniform to reduce randomness 'momentum': 0.98, 'momentum_scaling': 1, 'learning_rate_scaling': 100, 'batch_size': 128, })), ]), param_distributions={ 'nn__name': ['nn_{0:s}_{1:03d}'.format(name_from_file(), k) for k in range(10000)], 'nn__learning_rate': norm(0.0005, 0.0003), 'nn__dropout0_rate': triang(loc=0, c=0, scale=0.5), 'nn__dropout1_rate': uniform(loc=0.00, scale=0.50), 'nn__dropout2_rate': uniform(loc=0.15, scale=0.70 - 0.15), #'nn__dropout3_rate': uniform(loc = 0.30, scale = 0.80-0.30), #todo #'nn__weight_decay': norm(0.00006, 0.0001), }, fit_params={ 'nn__random_sleep': 80, }, n_iter=40,