def custom_experiment_parameters():
""" Here we define the experiment parameters.
We are using use_default_values=True, which will initialize
all the parameters with their default values. These parameters are then fixed
for the duration of the experiment and won't evolve.
That means that we need to manually specify which parametres we want to test,
and the possible values, either intervals or lists of values.
If we want to test all the parameters and possible values, we can
set use_default_values to False. In that case, random values will be generated
and tested during the experiment. We can redefine some parameters if we want to
fix their values.
Reference parameters and default values are defined in minos.model.parameters
"""
experiment_parameters = ExperimentParameters(use_default_values=True)
experiment_parameters.layout_parameter('rows', 1)
experiment_parameters.layout_parameter('blocks', int_param(1, 5))
experiment_parameters.layout_parameter('layers', int_param(1, 5))
experiment_parameters.layer_parameter('Dense.output_dim',
int_param(10, 100))
experiment_parameters.layer_parameter(
'Dense.activation',
string_param(['relu', 'tanh', 'custom_activation_1']))
experiment_parameters.layer_parameter('Dropout.p', float_param(0.1, 0.9))
return experiment_parameters
def test_mutate_w_custom_definitions(self):
def custom_activation(x):
return x
register_custom_activation('custom_activation', custom_activation)
register_custom_layer('Dense2', Dense, deepcopy(reference_parameters['layers']['Dense']))
layout = Layout(
input_size=100,
output_size=10,
output_activation='softmax',
block=['Dense', 'Dense2'])
training = Training(
objective=Objective('categorical_crossentropy'),
optimizer=None,
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(5),
batch_size=250)
experiment_parameters = ExperimentParameters(use_default_values=False)
experiment_parameters.layout_parameter('blocks', int_param(1, 5))
experiment_parameters.layout_parameter('layers', int_param(1, 5))
experiment_parameters.layer_parameter('Dense2.output_dim', int_param(10, 500))
experiment_parameters.layer_parameter('Dropout.p', float_param(0.1, 0.9))
experiment_parameters.all_search_parameters(True)
experiment = Experiment(
'test',
layout,
training,
batch_iterator=None,
test_batch_iterator=None,
environment=None,
parameters=experiment_parameters)
check_experiment_parameters(experiment)
for _ in range(10):
blueprint = create_random_blueprint(experiment)
mutant = mutate_blueprint(
blueprint,
parameters=experiment.parameters,
p_mutate_layout=0,
p_mutate_param=1,
mutate_in_place=False)
for row_idx, row in enumerate(mutant.layout.rows):
for block_idx, block in enumerate(row.blocks):
for layer_idx, layer in enumerate(block.layers):
original_row = blueprint.layout.rows[row_idx]
original_block = original_row.blocks[block_idx]
original_layer = original_block.layers[layer_idx]
for name, value in layer.parameters.items():
self.assertTrue(
value != original_layer.parameters[name],
'Should have mutated parameter')
def register_custom_definitions():
register_custom_activation('custom_activation_1', custom_activation)
register_custom_layer('custom_layer_1',
CustomLayer,
params={
'output_dim': int_param(10, 100),
'activation': 'custom_activation_1'
})
def test_custom_parameters(self):
experiment_parameters = ExperimentParameters()
experiment_parameters.layout_parameter('blocks', int_param(1, 10))
param = experiment_parameters.get_layout_parameter('blocks')
self.assertTrue(
1 == param.lo and 10 == param.hi,
'Should have set values')
experiment_parameters.layout_parameter('layers', int_param(1, 3))
param = experiment_parameters.get_layout_parameter('layers')
self.assertTrue(
1 == param.lo and 3 == param.hi,
'Should have set values')
experiment_parameters.layer_parameter('Dense.activation', string_param(['relu', 'tanh']))
param = experiment_parameters.get_layer_parameter('Dense.activation')
self.assertTrue(
'relu' == param.values[0] and 'tanh' == param.values[1],
'Should have set values')
def test_build(self):
layout = Layout(input_size=100,
output_size=10,
output_activation='softmax')
training = Training(objective=Objective('categorical_crossentropy'),
optimizer=None,
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(10),
batch_size=250)
experiment_parameters = ExperimentParameters(use_default_values=False)
experiment_parameters.layout_parameter('blocks', int_param(1, 5))
experiment_parameters.layout_parameter('layers', int_param(1, 5))
experiment_parameters.layer_parameter('Dense.output_dim',
int_param(10, 500))
experiment_parameters.layer_parameter('Dense.activation',
string_param(['relu', 'tanh']))
experiment_parameters.layer_parameter('Dropout.p',
float_param(0.1, 0.9))
experiment_parameters.all_search_parameters(True)
experiment = Experiment('test',
layout,
training,
batch_iterator=None,
test_batch_iterator=None,
environment=None,
parameters=experiment_parameters)
check_experiment_parameters(experiment)
for _ in range(5):
blueprint1 = create_random_blueprint(experiment)
model = ModelBuilder().build(blueprint1, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint2 = create_random_blueprint(experiment)
model = ModelBuilder().build(blueprint2, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint3 = mix_blueprints(blueprint1, blueprint2,
experiment_parameters)
model = ModelBuilder().build(blueprint3, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint4 = mutate_blueprint(blueprint1,
experiment_parameters,
mutate_in_place=False)
model = ModelBuilder().build(blueprint4, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
def test_random_value(self):
param = int_param(values=list(range(10)))
val = random_param_value(param)
self.assertTrue(
isinstance(val, int),
'Should be an int')
self.assertTrue(
val in param.values,
'Value should be in predefined values')
param = float_param(values=[i * 0.1 for i in range(10)])
val = random_param_value(param)
self.assertTrue(
isinstance(val, float),
'Should be a float')
self.assertTrue(
val in param.values,
'Value should be in predefined values')
param = float_param(lo=.5, hi=.7)
for _ in range(100):
val = random_param_value(param)
self.assertTrue(
isinstance(val, float),
'Should be a float')
self.assertTrue(
val <= param.hi and val >= param.lo,
'Value should be in range')
param = {
'a': float_param(optional=False),
'b': float_param(optional=False)}
for _ in range(10):
val = random_param_value(param)
self.assertTrue(
isinstance(val, dict),
'Should be a dict')
self.assertEqual(
len(param), len(val),
'Should respect non optional setting')
param = {
'a': float_param(optional=True),
'b': float_param(optional=True)}
for _ in range(10):
val = random_param_value(param)
self.assertTrue(
isinstance(val, dict),
'Should be a dict')
self.assertTrue(
len(val) >= 0 and len(val) <= len(param),
'Should respect non optional setting')
def test_save(self):
disable_sysout()
def custom_activation(x):
return x
register_custom_activation('custom_activation', custom_activation)
register_custom_layer('custom_layer', CustomLayer,
{'output_dim': int_param(1, 100)})
with tempfile.TemporaryDirectory() as tmp_dir:
batch_size = 50
batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(
batch_size, 1000)
layout = Layout(input_size=1000,
output_size=nb_classes,
output_activation='softmax')
training = Training(
objective=Objective('categorical_crossentropy'),
optimizer=Optimizer(optimizer='Adam'),
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(10),
batch_size=batch_size)
experiment_parameters = ExperimentParameters(
use_default_values=True)
experiment_parameters.layout_parameter('rows', 1)
experiment_parameters.layout_parameter('blocks', 1)
experiment_parameters.layout_parameter('layers', 1)
experiment_parameters.layout_parameter('block.layer_type',
'custom_layer')
experiment = Experiment('test__reuters_experiment',
layout,
training,
batch_iterator,
test_batch_iterator,
CpuEnvironment(n_jobs=1, data_dir=tmp_dir),
parameters=experiment_parameters)
blueprint = create_random_blueprint(experiment)
model = ModelBuilder().build(blueprint, default_device())
model.fit_generator(
generator=batch_iterator,
samples_per_epoch=batch_iterator.samples_per_epoch,
nb_epoch=10,
validation_data=test_batch_iterator,
nb_val_samples=test_batch_iterator.sample_count)
filepath = join(tmp_dir, 'model')
model.save(filepath)
model = load_keras_model(filepath)
self.assertIsNotNone(model, 'Should have loaded the model')
def test_build_w_custom_definitions(self):
def custom_activation(x):
return x
register_custom_activation('custom_activation', custom_activation)
register_custom_layer(
'Dense2', Dense, deepcopy(reference_parameters['layers']['Dense']),
True)
layout = Layout(input_size=100,
output_size=10,
output_activation='softmax',
block=['Dense2'])
training = Training(objective=Objective('categorical_crossentropy'),
optimizer=None,
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(5),
batch_size=250)
experiment_parameters = ExperimentParameters(use_default_values=False)
experiment_parameters.layout_parameter('blocks', int_param(1, 5))
experiment_parameters.layout_parameter('layers', int_param(1, 5))
experiment_parameters.layout_parameter('layer.type',
string_param(['Dense2']))
experiment_parameters.layer_parameter('Dense2.output_dim',
int_param(10, 500))
experiment_parameters.layer_parameter(
'Dense2.activation', string_param(['custom_activation']))
experiment_parameters.layer_parameter('Dropout.p',
float_param(0.1, 0.9))
experiment_parameters.all_search_parameters(True)
experiment = Experiment('test',
layout,
training,
batch_iterator=None,
test_batch_iterator=None,
environment=None,
parameters=experiment_parameters)
check_experiment_parameters(experiment)
for _ in range(5):
blueprint1 = create_random_blueprint(experiment)
for layer in blueprint1.layout.get_layers():
self.assertEqual('Dense2', layer.layer_type,
'Should have used custom layer')
model = ModelBuilder().build(blueprint1, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint2 = create_random_blueprint(experiment)
for layer in blueprint2.layout.get_layers():
self.assertEqual('Dense2', layer.layer_type,
'Should have used custom layer')
model = ModelBuilder().build(blueprint2, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint3 = mix_blueprints(blueprint1, blueprint2,
experiment_parameters)
for layer in blueprint3.layout.get_layers():
self.assertEqual('Dense2', layer.layer_type,
'Should have used custom layer')
model = ModelBuilder().build(blueprint3, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint4 = mutate_blueprint(blueprint1,
experiment_parameters,
mutate_in_place=False)
for layer in blueprint4.layout.get_layers():
self.assertEqual('Dense2', layer.layer_type,
'Should have used custom layer')
model = ModelBuilder().build(blueprint4, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
Created on Feb 14, 2017
@author: julien
'''
from minos.model.parameter import int_param, string_param, boolean_param,\
float_param
reference_parameters = {
'search': {
'layout': boolean_param(default=True),
'parameters': boolean_param(default=True),
'optimizer': boolean_param(default=True)
},
'layout': {
'rows': int_param(lo=1, hi=3, default=1),
'blocks': int_param(lo=1, hi=5, default=1),
'layers': int_param(lo=1, hi=5, default=1),
'block': {
'input_type':
string_param(['concat', 'random+concat', 'concat+random'],
default='concat'),
'input_size':
float_param(default=1.)
},
'layer': {
'type': string_param(['Dense', 'Dropout', 'BatchNormalization']),
'stackable': string_param([])
}
},
'layers': {
def test_ga_search(self):
with tempfile.TemporaryDirectory() as tmp_dir:
epoch = 3
generations = 2
batch_size = 50
batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(
batch_size, 1000)
layout = Layout(input_size=1000,
output_size=nb_classes,
output_activation='softmax')
training = Training(
objective=Objective('categorical_crossentropy'),
optimizer=Optimizer(optimizer='Adam'),
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(epoch),
batch_size=batch_size)
experiment_parameters = ExperimentParameters(
use_default_values=False)
experiment_parameters.layout_parameter('rows', 1)
experiment_parameters.layout_parameter('blocks', 1)
experiment_parameters.layout_parameter('layers', 1)
experiment_parameters.layer_parameter('Dense.output_dim',
int_param(10, 500))
experiment_parameters.all_search_parameters(True)
experiment_label = 'test__reuters_experiment'
experiment = Experiment(experiment_label,
layout,
training,
batch_iterator,
test_batch_iterator,
CpuEnvironment(n_jobs=2, data_dir=tmp_dir),
parameters=experiment_parameters)
run_ga_search_experiment(experiment,
population_size=2,
generations=2)
self.assertTrue(isfile(experiment.get_log_filename()),
'Should have logged')
self.assertTrue(isfile(experiment.get_step_data_filename(0)),
'Should have logged')
self.assertTrue(isfile(experiment.get_step_log_filename(0)),
'Should have logged')
blueprints = load_experiment_blueprints(
experiment_label, 0, Environment(data_dir=tmp_dir))
self.assertTrue(
len(blueprints) > 0, 'Should have saved/loaded blueprints')
model = ModelBuilder().build(blueprints[0], cpu_device())
disable_sysout()
model.fit_generator(
generator=batch_iterator,
samples_per_epoch=batch_iterator.samples_per_epoch,
nb_epoch=5,
validation_data=test_batch_iterator,
nb_val_samples=test_batch_iterator.sample_count)
score = model.evaluate_generator(
test_batch_iterator,
val_samples=test_batch_iterator.sample_count)
self.assertTrue(score[1] > 0, 'Should have valid score')
step, population = load_experiment_checkpoint(experiment)
self.assertEqual(generations - 1, step,
'Should have loaded checkpoint')
self.assertIsNotNone(population, 'Should have loaded checkpoint')
blueprint = load_experiment_best_blueprint(
experiment.label,
environment=CpuEnvironment(n_jobs=2, data_dir=tmp_dir))
model = ModelBuilder().build(blueprint,
cpu_device(),
compile_model=False)
self.assertIsNotNone(
model,
'Should have loaded and built best model from experiment')