def tests():
over_id = id_gen('overfit')
base = {
'initial_learning_rate': 0.01,
'decay_factor': 0.99,
'regularization': 0,
'target': 'overfit'
}
# One layer
yield create_and_train([], False, momentum=.8, train_id=next(over_id), **base)
# Two layers
# With or without batch norm
for bn in [False, True]:
# Different momentum values
for m in [.5, .7, .9]:
yield create_and_train([50], batch_normalized=bn, momentum=m,
train_id=next(over_id), **base)
# Three layers
# With or without batch norm
for bn in [False, True]:
yield create_and_train([50, 30], batch_normalized=bn, momentum=.8,
train_id=next(over_id), **base)
# Four layers
# With or without batch norm
for bn in [False, True]:
yield create_and_train([50, 30, 15], batch_normalized=bn, momentum=.8,
train_id=next(over_id), **base)
def tests():
deep_id = id_gen('deep')
base = {
'hidden_layer_sizes': [50, 30],
'decay_factor': 0.99,
'momentum': 0.8
}
for bn in [False, True]:
yield create_and_train(batch_normalized=bn,
regularization=0,
initial_learning_rate=0.01,
**base,
train_id=next(deep_id),
target='fine')
# Coarse search with batch norm (random values)
for reg in [0.000192, 0.000672, 0.003772, 0.032430, 0.266132]:
for ilr in [0.40573, 0.029202, 0.012721, 0.003215]:
yield create_and_train(batch_normalized=True,
regularization=reg,
initial_learning_rate=ilr,
**base,
train_id=next(deep_id),
target='coarse')
# Fine search with batch norm
for reg in [0.012361, 0.032430, 0.053913]:
for ilr in [0.043773, 0.029202, 0.016231, 0.001051]:
yield create_and_train(batch_normalized=True,
regularization=reg,
initial_learning_rate=ilr,
**base,
train_id=next(deep_id),
target='fine')
# Last one
yield create_and_train(batch_normalized=True,
regularization=0.0001,
initial_learning_rate=0.029202,
**base,
train_id=next(deep_id),
target='final')
def tests():
fast_id = id_gen('fast')
base = {
'hidden_layer_sizes': [50],
'decay_factor': 0.99,
'regularization': 0.0001,
'momentum': 0.8,
'target': 'fine'
}
for bn in [False, True]:
for ilr in [0.40573, 0.029202, 0.003215]:
yield create_and_train(batch_normalized=bn,
initial_learning_rate=ilr,
**base,
train_id=next(fast_id))
def best_net():
training = cifar.get_named_batches('data_batch_1', 'data_batch_2',
'data_batch_3', 'data_batch_4',
'data_batch_5')
validation = training
test = cifar.get_named_batches('test_batch')
result = create_and_train(training,
validation,
epochs=50,
hidden_size=50,
regularization=0.00275,
initial_learning_rate=.35,
decay_factor=0.998,
momentum=.8,
train_id='final',
test=test)
pd.DataFrame([result]).to_csv('final.csv', mode='a', header=False)
def fine_search():
# Parameter search space
regularization = np.linspace(0.005, 0.0005, num=7)
initial_learning_rate = np.linspace(0.35, 0.25, num=4)
parameters = list(itertools.product(regularization, initial_learning_rate))
# Training
for i, p in enumerate(tqdm(parameters, desc='Fine search')):
result = create_and_train(training,
validation,
epochs=10,
hidden_size=50,
regularization=p[0],
initial_learning_rate=p[1],
decay_factor=0.998,
momentum=.8,
train_id='fine_{}'.format(i))
pd.DataFrame([result]).to_csv('fine.csv', mode='a', header=False)
def coarse_search():
# Parameter search space
regularization = [0.001, 0.005, 0.01, 0.05, 0.1, 0.3, 0.5]
initial_learning_rate = [0.5, 0.3, 0.1, 0.05, 0.01, 0.005, 0.001]
parameters = list(itertools.product(regularization, initial_learning_rate))
# Training
for i, p in enumerate(tqdm(parameters, desc='Coarse search')):
result = create_and_train(training,
validation,
epochs=5,
hidden_size=50,
regularization=p[0],
initial_learning_rate=p[1],
decay_factor=0.998,
momentum=.8,
train_id='coarse_{}'.format(i))
pd.DataFrame([result]).to_csv('coarse.csv', mode='a', header=False)
def simple_create_and_train(regularization, learning_rate, iterations,
plot_name):
return create_and_train(training, validation, iterations, 50,
regularization, learning_rate, 0.99, 0.8,
plot_name)