def multiprocess_fit(blueprint):
disable_sysout()
from minos.model.build import ModelBuilder
model = ModelBuilder().build(blueprint, default_device())
model.fit_generator(generator=batch_iterator,
samples_per_epoch=batch_iterator.samples_per_epoch,
nb_epoch=10,
validation_data=test_batch_iterator,
nb_val_samples=test_batch_iterator.sample_count)
def load_best_model(experiment_label, step):
""" Here we load the blueprints generated during an experiment
and create the Keras model from the top scoring blueprint
"""
blueprint = load_experiment_best_blueprint(experiment_label, step,
Environment())
return ModelBuilder().build(blueprint, cpu_device(), compile_model=False)
def test_build(self):
layout = Layout(input_size=100,
output_size=10,
output_activation='softmax')
training = Training(objective=Objective('categorical_crossentropy'),
optimizer=None,
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(10),
batch_size=250)
experiment_parameters = ExperimentParameters(use_default_values=False)
experiment_parameters.layout_parameter('blocks', int_param(1, 5))
experiment_parameters.layout_parameter('layers', int_param(1, 5))
experiment_parameters.layer_parameter('Dense.output_dim',
int_param(10, 500))
experiment_parameters.layer_parameter('Dense.activation',
string_param(['relu', 'tanh']))
experiment_parameters.layer_parameter('Dropout.p',
float_param(0.1, 0.9))
experiment_parameters.all_search_parameters(True)
experiment = Experiment('test',
layout,
training,
batch_iterator=None,
test_batch_iterator=None,
environment=None,
parameters=experiment_parameters)
check_experiment_parameters(experiment)
for _ in range(5):
blueprint1 = create_random_blueprint(experiment)
model = ModelBuilder().build(blueprint1, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint2 = create_random_blueprint(experiment)
model = ModelBuilder().build(blueprint2, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint3 = mix_blueprints(blueprint1, blueprint2,
experiment_parameters)
model = ModelBuilder().build(blueprint3, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint4 = mutate_blueprint(blueprint1,
experiment_parameters,
mutate_in_place=False)
model = ModelBuilder().build(blueprint4, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
def test_train(self):
disable_sysout()
with tempfile.TemporaryDirectory() as tmp_dir:
batch_size = 50
batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(
batch_size, 1000)
layout = Layout(input_size=1000,
output_size=nb_classes,
output_activation='softmax')
training = Training(
objective=Objective('categorical_crossentropy'),
optimizer=Optimizer(optimizer='Adam'),
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(10),
batch_size=batch_size)
experiment_parameters = ExperimentParameters(
use_default_values=True)
experiment_parameters.layout_parameter('rows', 1)
experiment_parameters.layout_parameter('blocks', 1)
experiment_parameters.layout_parameter('layers', 1)
experiment = Experiment('test__reuters_experiment',
layout,
training,
batch_iterator,
test_batch_iterator,
CpuEnvironment(n_jobs=1, data_dir=tmp_dir),
parameters=experiment_parameters)
blueprint = create_random_blueprint(experiment)
model = ModelBuilder().build(blueprint, default_device())
result = model.fit_generator(
generator=batch_iterator,
samples_per_epoch=batch_iterator.samples_per_epoch,
nb_epoch=10,
validation_data=test_batch_iterator,
nb_val_samples=test_batch_iterator.sample_count)
self.assertIsNotNone(result, 'should have fit the model')
score = model.evaluate_generator(
test_batch_iterator,
val_samples=test_batch_iterator.sample_count)
self.assertIsNotNone(score, 'should have evaluated the model')
def test_save(self):
disable_sysout()
def custom_activation(x):
return x
register_custom_activation('custom_activation', custom_activation)
register_custom_layer('custom_layer', CustomLayer,
{'output_dim': int_param(1, 100)})
with tempfile.TemporaryDirectory() as tmp_dir:
batch_size = 50
batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(
batch_size, 1000)
layout = Layout(input_size=1000,
output_size=nb_classes,
output_activation='softmax')
training = Training(
objective=Objective('categorical_crossentropy'),
optimizer=Optimizer(optimizer='Adam'),
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(10),
batch_size=batch_size)
experiment_parameters = ExperimentParameters(
use_default_values=True)
experiment_parameters.layout_parameter('rows', 1)
experiment_parameters.layout_parameter('blocks', 1)
experiment_parameters.layout_parameter('layers', 1)
experiment_parameters.layout_parameter('block.layer_type',
'custom_layer')
experiment = Experiment('test__reuters_experiment',
layout,
training,
batch_iterator,
test_batch_iterator,
CpuEnvironment(n_jobs=1, data_dir=tmp_dir),
parameters=experiment_parameters)
blueprint = create_random_blueprint(experiment)
model = ModelBuilder().build(blueprint, default_device())
model.fit_generator(
generator=batch_iterator,
samples_per_epoch=batch_iterator.samples_per_epoch,
nb_epoch=10,
validation_data=test_batch_iterator,
nb_val_samples=test_batch_iterator.sample_count)
filepath = join(tmp_dir, 'model')
model.save(filepath)
model = load_keras_model(filepath)
self.assertIsNotNone(model, 'Should have loaded the model')
def test_build_w_custom_definitions(self):
def custom_activation(x):
return x
register_custom_activation('custom_activation', custom_activation)
register_custom_layer(
'Dense2', Dense, deepcopy(reference_parameters['layers']['Dense']),
True)
layout = Layout(input_size=100,
output_size=10,
output_activation='softmax',
block=['Dense2'])
training = Training(objective=Objective('categorical_crossentropy'),
optimizer=None,
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(5),
batch_size=250)
experiment_parameters = ExperimentParameters(use_default_values=False)
experiment_parameters.layout_parameter('blocks', int_param(1, 5))
experiment_parameters.layout_parameter('layers', int_param(1, 5))
experiment_parameters.layout_parameter('layer.type',
string_param(['Dense2']))
experiment_parameters.layer_parameter('Dense2.output_dim',
int_param(10, 500))
experiment_parameters.layer_parameter(
'Dense2.activation', string_param(['custom_activation']))
experiment_parameters.layer_parameter('Dropout.p',
float_param(0.1, 0.9))
experiment_parameters.all_search_parameters(True)
experiment = Experiment('test',
layout,
training,
batch_iterator=None,
test_batch_iterator=None,
environment=None,
parameters=experiment_parameters)
check_experiment_parameters(experiment)
for _ in range(5):
blueprint1 = create_random_blueprint(experiment)
for layer in blueprint1.layout.get_layers():
self.assertEqual('Dense2', layer.layer_type,
'Should have used custom layer')
model = ModelBuilder().build(blueprint1, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint2 = create_random_blueprint(experiment)
for layer in blueprint2.layout.get_layers():
self.assertEqual('Dense2', layer.layer_type,
'Should have used custom layer')
model = ModelBuilder().build(blueprint2, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint3 = mix_blueprints(blueprint1, blueprint2,
experiment_parameters)
for layer in blueprint3.layout.get_layers():
self.assertEqual('Dense2', layer.layer_type,
'Should have used custom layer')
model = ModelBuilder().build(blueprint3, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
blueprint4 = mutate_blueprint(blueprint1,
experiment_parameters,
mutate_in_place=False)
for layer in blueprint4.layout.get_layers():
self.assertEqual('Dense2', layer.layer_type,
'Should have used custom layer')
model = ModelBuilder().build(blueprint4, cpu_device())
self.assertIsNotNone(model, 'Should have built a model')
def __init__(self, batch_iterator, test_batch_iterator):
from minos.model.build import ModelBuilder
self.model_builder = ModelBuilder()
self.batch_iterator = batch_iterator
self.test_batch_iterator = test_batch_iterator
class ModelTrainer(object):
def __init__(self, batch_iterator, test_batch_iterator):
from minos.model.build import ModelBuilder
self.model_builder = ModelBuilder()
self.batch_iterator = batch_iterator
self.test_batch_iterator = test_batch_iterator
def train(self,
blueprint,
device,
save_best_model=False,
model_filename=None):
try:
model = self.model_builder.build(blueprint, device)
setup_tf_session(device)
nb_epoch, callbacks = self._get_stopping_parameters(blueprint)
if save_best_model:
callbacks.append(
self._get_model_save_callback(
model_filename, blueprint.training.metric.metric))
start = time()
history = model.fit_generator(
self.batch_iterator,
self.batch_iterator.samples_per_epoch,
nb_epoch,
callbacks=callbacks,
validation_data=self.test_batch_iterator,
nb_val_samples=self.test_batch_iterator.sample_count)
if save_best_model:
del model
model = load_keras_model(model_filename)
return model, history, (time() - start)
except Exception as ex:
logging.debug(ex)
logging.debug(traceback.format_exc())
try:
from keras import backend
backend.clear_session()
except:
logging.debug(ex)
logging.debug(traceback.format_exc())
return None, None, 0
def _get_model_save_callback(self, model_filename, metric):
checkpoint = ModelCheckpoint(model_filename,
monitor=metric,
save_best_only=True)
return checkpoint
def _get_stopping_parameters(self, blueprint):
if isinstance(blueprint.training.stopping, EpochStoppingCondition):
nb_epoch = blueprint.training.stopping.epoch
stopping_callbacks = []
if isinstance(blueprint.training.stopping,
AccuracyDecreaseStoppingCondition):
nb_epoch = max(1, blueprint.training.stopping.min_epoch,
blueprint.training.stopping.max_epoch)
stopping_callbacks = [
AccuracyDecreaseStoppingConditionWrapper(
blueprint.training.stopping)
]
return nb_epoch, stopping_callbacks
def test_ga_search(self):
with tempfile.TemporaryDirectory() as tmp_dir:
epoch = 3
generations = 2
batch_size = 50
batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(
batch_size, 1000)
layout = Layout(input_size=1000,
output_size=nb_classes,
output_activation='softmax')
training = Training(
objective=Objective('categorical_crossentropy'),
optimizer=Optimizer(optimizer='Adam'),
metric=Metric('categorical_accuracy'),
stopping=EpochStoppingCondition(epoch),
batch_size=batch_size)
experiment_parameters = ExperimentParameters(
use_default_values=False)
experiment_parameters.layout_parameter('rows', 1)
experiment_parameters.layout_parameter('blocks', 1)
experiment_parameters.layout_parameter('layers', 1)
experiment_parameters.layer_parameter('Dense.output_dim',
int_param(10, 500))
experiment_parameters.all_search_parameters(True)
experiment_label = 'test__reuters_experiment'
experiment = Experiment(experiment_label,
layout,
training,
batch_iterator,
test_batch_iterator,
CpuEnvironment(n_jobs=2, data_dir=tmp_dir),
parameters=experiment_parameters)
run_ga_search_experiment(experiment,
population_size=2,
generations=2)
self.assertTrue(isfile(experiment.get_log_filename()),
'Should have logged')
self.assertTrue(isfile(experiment.get_step_data_filename(0)),
'Should have logged')
self.assertTrue(isfile(experiment.get_step_log_filename(0)),
'Should have logged')
blueprints = load_experiment_blueprints(
experiment_label, 0, Environment(data_dir=tmp_dir))
self.assertTrue(
len(blueprints) > 0, 'Should have saved/loaded blueprints')
model = ModelBuilder().build(blueprints[0], cpu_device())
disable_sysout()
model.fit_generator(
generator=batch_iterator,
samples_per_epoch=batch_iterator.samples_per_epoch,
nb_epoch=5,
validation_data=test_batch_iterator,
nb_val_samples=test_batch_iterator.sample_count)
score = model.evaluate_generator(
test_batch_iterator,
val_samples=test_batch_iterator.sample_count)
self.assertTrue(score[1] > 0, 'Should have valid score')
step, population = load_experiment_checkpoint(experiment)
self.assertEqual(generations - 1, step,
'Should have loaded checkpoint')
self.assertIsNotNone(population, 'Should have loaded checkpoint')
blueprint = load_experiment_best_blueprint(
experiment.label,
environment=CpuEnvironment(n_jobs=2, data_dir=tmp_dir))
model = ModelBuilder().build(blueprint,
cpu_device(),
compile_model=False)
self.assertIsNotNone(
model,
'Should have loaded and built best model from experiment')