def test_mutate_layout(self): layout = Layout(input_size=100, output_size=10, output_activation='softmax') training = Training(objective=None, optimizer=None, metric=None, stopping=None, batch_size=None) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=ExperimentParameters(use_default_values=False)) experiment.parameters.all_search_parameters(True) check_experiment_parameters(experiment) for _ in range(10): blueprint1 = create_random_blueprint(experiment) blueprint2 = create_random_blueprint(experiment) mutant = mix_blueprints(blueprint1, blueprint2, parameters=experiment.parameters, p_mutate_param=0.1) blueprints = [blueprint1, blueprint2] parent_rows = [len(b.layout.rows) for b in blueprints] self.assertTrue( len(mutant.layout.rows) in parent_rows, 'Should have used one of the parents')
def test_predefined_multiple_blocklayout(self): layout = Layout( input_size=100, output_size=10, output_activation='softmax', block=[ [('Dense', {'activation': 'relu'})], [('Dense', {'activation': 'relu'}), 'Dropout', ('Dense', {'output_dim': 100})]]) training = Training( objective=None, optimizer=Optimizer(), metric=None, stopping=None, batch_size=None) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=ExperimentParameters(use_default_values=False)) experiment.parameters.search_parameter('layout', False) experiment.parameters.search_parameter('parameters', True) experiment.parameters.search_parameter('optimizer', True) for _ in range(10): blueprint1 = create_random_blueprint(experiment) blueprint2 = create_random_blueprint(experiment) blueprint3 = mix_blueprints(blueprint1, blueprint2, experiment.parameters) blueprint4 = mutate_blueprint(blueprint1, experiment.parameters, mutate_in_place=False) for idx, blueprint in enumerate([blueprint1, blueprint2, blueprint3, blueprint4]): self.assertIsNotNone(blueprint, 'Should have created a blueprint') self.assertIsNotNone(blueprint.layout, 'Should have created a layout') self.assertEqual( 1, len(blueprint.layout.get_rows()), 'Should have 1 row') self.assertEqual( 1, len(blueprint.layout.get_blocks()), 'Should have 1 block') for i, row in enumerate(blueprint.layout.get_rows()): blocks = len(row.get_blocks()) self.assertTrue( is_valid_param_value( experiment.parameters.get_layout_parameter('blocks'), blocks), 'Invalid value') for block in row.get_blocks(): self.assertTrue( len(block.layers) == len(layout.block[0]) or len(block.layers) == len(layout.block[1]), 'Should have used template')
def test_predefined_layer_type(self): layout = Layout( input_size=100, output_size=10, output_activation='softmax') training = Training( objective=None, optimizer=Optimizer('SGD', {'lr': 1}), metric=None, stopping=None, batch_size=None) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None) layer_types = ['Dropout'] experiment.parameters.layer_types(string_param(layer_types)) for _ in range(10): blueprint = create_random_blueprint(experiment) self.assertIsNotNone(blueprint, 'Should have created a blueprint') for layer in blueprint.layout.get_layers(): self.assertTrue( layer.layer_type in layer_types, 'Should have used predefined layer types')
def test_train(self): n_jobs = 2 with Pool(n_jobs) as pool,\ tempfile.TemporaryDirectory() as tmp_dir: layout = Layout(input_size=1000, output_size=nb_classes, output_activation='softmax') training = Training( objective=Objective('categorical_crossentropy'), optimizer=Optimizer(optimizer='Adam'), metric=Metric('categorical_accuracy'), stopping=EpochStoppingCondition(10), batch_size=batch_size) experiment_parameters = ExperimentParameters( use_default_values=True) experiment_parameters.layout_parameter('rows', 1) experiment_parameters.layout_parameter('blocks', 1) experiment_parameters.layout_parameter('layers', 1) experiment = Experiment('test__reuters_experiment', layout, training, batch_iterator, test_batch_iterator, CpuEnvironment(n_jobs=1, data_dir=tmp_dir), parameters=experiment_parameters) blueprints = [ create_random_blueprint(experiment) for _ in range(n_jobs) ] pool.map(multiprocess_fit, blueprints)
def train_multi_gpu(max_words=1000, batch_size=32): batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset( batch_size, max_words) experiment = create_experiment(max_words, nb_classes, batch_size) blueprint = create_random_blueprint(experiment) devices = ['/gpu:0', '/gpu:1'] trainer = ModelTrainer(batch_iterator, test_batch_iterator) with tempfile.TemporaryDirectory() as tmp_dir: model, history, _duration = trainer.train(blueprint, devices, save_best_model=True, model_filename=join( tmp_dir, 'model')) metric = get_associated_validation_metric( blueprint.training.metric.metric) epoch = numpy.argmax(history.history[metric]) score = history.history[metric][epoch] print('Final training score %r after %d epoch' % (score, epoch)) test_size = 10 y_true = numpy.argmax(test_batch_iterator.y[0][:test_size], axis=1) y_pred = numpy.argmax(model.predict( test_batch_iterator.X[0][:test_size]), axis=1) evaluation = numpy.mean(y_true == y_pred) print('Final evaluation score %f' % evaluation) print('Predictions (true, pred) %r' % list(zip(y_true.tolist(), y_pred.tolist())))
def test_predefined_parameters(self): layout = Layout( input_size=100, output_size=10, output_activation='softmax') training = Training( objective=None, optimizer=Optimizer('SGD', {'lr': 1}), metric=None, stopping=None, batch_size=None) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None) for _ in range(10): blueprint = create_random_blueprint(experiment) self.assertIsNotNone(blueprint, 'Should have created a blueprint') self.assertEqual( training.optimizer.optimizer, blueprint.training.optimizer.optimizer, 'Should have created an optimizer') self.assertEqual( blueprint.training.optimizer.parameters['lr'], training.optimizer.parameters['lr'], 'Should have copied predefined parameter')
def test_random_blueprint(self): layout = Layout( input_size=100, output_size=10, output_activation='softmax') training = Training( objective=None, optimizer=None, metric=None, stopping=None, batch_size=None) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=ExperimentParameters(use_default_values=False)) experiment.parameters.all_search_parameters(True) check_experiment_parameters(experiment) for _ in range(10): blueprint = create_random_blueprint(experiment) self.assertIsNotNone(blueprint, 'Should have created a blueprint') optimizer = blueprint.training.optimizer self.assertIsNotNone( optimizer.optimizer, 'Should have created an optimizer') ref_parameters = experiment.parameters.get_optimizers_parameters() for name, param in ref_parameters[optimizer.optimizer].items(): self.assertTrue( is_valid_param_value(param, optimizer.parameters[name]), 'Invalid param value') self.assertIsNotNone(blueprint.layout, 'Should have created a layout') rows = len(blueprint.layout.get_rows()) self.assertTrue( is_valid_param_value( experiment.parameters.get_layout_parameter('rows'), rows), 'Invalid value') for row in blueprint.layout.get_rows(): blocks = len(row.get_blocks()) self.assertTrue( is_valid_param_value( experiment.parameters.get_layout_parameter('blocks'), blocks), 'Invalid value') for block in row.get_blocks(): layers = len([ l for l in block.get_layers() if l.layer_type != 'Merge']) self.assertTrue( is_valid_param_value( experiment.parameters.get_layout_parameter('layers'), layers), 'Invalid value')
def test_build(self): layout = Layout(input_size=100, output_size=10, output_activation='softmax') training = Training(objective=Objective('categorical_crossentropy'), optimizer=None, metric=Metric('categorical_accuracy'), stopping=EpochStoppingCondition(10), batch_size=250) experiment_parameters = ExperimentParameters(use_default_values=False) experiment_parameters.layout_parameter('blocks', int_param(1, 5)) experiment_parameters.layout_parameter('layers', int_param(1, 5)) experiment_parameters.layer_parameter('Dense.output_dim', int_param(10, 500)) experiment_parameters.layer_parameter('Dense.activation', string_param(['relu', 'tanh'])) experiment_parameters.layer_parameter('Dropout.p', float_param(0.1, 0.9)) experiment_parameters.all_search_parameters(True) experiment = Experiment('test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=experiment_parameters) check_experiment_parameters(experiment) for _ in range(5): blueprint1 = create_random_blueprint(experiment) model = ModelBuilder().build(blueprint1, cpu_device()) self.assertIsNotNone(model, 'Should have built a model') blueprint2 = create_random_blueprint(experiment) model = ModelBuilder().build(blueprint2, cpu_device()) self.assertIsNotNone(model, 'Should have built a model') blueprint3 = mix_blueprints(blueprint1, blueprint2, experiment_parameters) model = ModelBuilder().build(blueprint3, cpu_device()) self.assertIsNotNone(model, 'Should have built a model') blueprint4 = mutate_blueprint(blueprint1, experiment_parameters, mutate_in_place=False) model = ModelBuilder().build(blueprint4, cpu_device()) self.assertIsNotNone(model, 'Should have built a model')
def test_mutate_layout(self): layout = Layout( input_size=100, output_size=10, output_activation='softmax') training = Training( objective=None, optimizer=None, metric=None, stopping=None, batch_size=None) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=ExperimentParameters(use_default_values=False)) experiment.parameters.all_search_parameters(True) check_experiment_parameters(experiment) for _ in range(10): blueprint = create_random_blueprint(experiment) mutant = mutate_blueprint( blueprint, parameters=experiment.parameters, p_mutate_layout=1, layout_mutation_count=1, layout_mutables=['rows'], mutate_in_place=False) self.assertTrue( len(mutant.layout.rows) != len(blueprint.layout.rows), 'Should have mutated rows') mutant = mutate_blueprint( blueprint, parameters=experiment.parameters, p_mutate_layout=1, layout_mutation_count=1, layout_mutables=['blocks'], mutate_in_place=False) self.assertTrue( len(mutant.layout.get_blocks()) != len(blueprint.layout.get_blocks()), 'Should have mutated blocks') mutant = mutate_blueprint( blueprint, parameters=experiment.parameters, p_mutate_layout=1, layout_mutation_count=1, layout_mutables=['layers'], mutate_in_place=False) self.assertTrue( len(mutant.layout.get_layers()) != len(blueprint.layout.get_layers()), 'Should have mutated layers')
def test_mutate_w_custom_definitions(self): def custom_activation(x): return x register_custom_activation('custom_activation', custom_activation) register_custom_layer('Dense2', Dense, deepcopy(reference_parameters['layers']['Dense'])) layout = Layout( input_size=100, output_size=10, output_activation='softmax', block=['Dense', 'Dense2']) training = Training( objective=Objective('categorical_crossentropy'), optimizer=None, metric=Metric('categorical_accuracy'), stopping=EpochStoppingCondition(5), batch_size=250) experiment_parameters = ExperimentParameters(use_default_values=False) experiment_parameters.layout_parameter('blocks', int_param(1, 5)) experiment_parameters.layout_parameter('layers', int_param(1, 5)) experiment_parameters.layer_parameter('Dense2.output_dim', int_param(10, 500)) experiment_parameters.layer_parameter('Dropout.p', float_param(0.1, 0.9)) experiment_parameters.all_search_parameters(True) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=experiment_parameters) check_experiment_parameters(experiment) for _ in range(10): blueprint = create_random_blueprint(experiment) mutant = mutate_blueprint( blueprint, parameters=experiment.parameters, p_mutate_layout=0, p_mutate_param=1, mutate_in_place=False) for row_idx, row in enumerate(mutant.layout.rows): for block_idx, block in enumerate(row.blocks): for layer_idx, layer in enumerate(block.layers): original_row = blueprint.layout.rows[row_idx] original_block = original_row.blocks[block_idx] original_layer = original_block.layers[layer_idx] for name, value in layer.parameters.items(): self.assertTrue( value != original_layer.parameters[name], 'Should have mutated parameter')
def test_early_stopping_condition_test(self): disable_sysout() with tempfile.TemporaryDirectory() as tmp_dir: batch_size = 50 min_epoch = 10 max_epoch = 15 batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset( batch_size, 1000) layout = Layout(input_size=1000, output_size=nb_classes, output_activation='softmax') training = Training( objective=Objective('categorical_crossentropy'), optimizer=Optimizer(optimizer='Adam'), metric=Metric('categorical_accuracy'), stopping=AccuracyDecreaseStoppingCondition( metric='categorical_accuracy', noprogress_count=2, min_epoch=min_epoch, max_epoch=max_epoch), batch_size=batch_size) experiment_parameters = ExperimentParameters( use_default_values=True) experiment_parameters.layout_parameter('rows', 1) experiment_parameters.layout_parameter('blocks', 1) experiment_parameters.layout_parameter('layers', 1) experiment = Experiment('test__reuters_experiment', layout, training, batch_iterator, test_batch_iterator, CpuEnvironment(n_jobs=1, data_dir=tmp_dir), parameters=experiment_parameters) _assert_valid_training_parameters(experiment) blueprint = create_random_blueprint(experiment) trainer = ModelTrainer(batch_iterator, test_batch_iterator) model, history, _duration = trainer.train(blueprint, cpu_device(), save_best_model=False) self.assertTrue( len(history.epoch) >= min_epoch, 'Should have trained for at least min epoch') self.assertTrue( len(history.epoch) <= max_epoch, 'Should have trained for max epoch') self.assertIsNotNone(model, 'should have fit the model') score = model.evaluate_generator( test_batch_iterator, val_samples=test_batch_iterator.sample_count) self.assertIsNotNone(score, 'should have evaluated the model')
def test_save(self): disable_sysout() def custom_activation(x): return x register_custom_activation('custom_activation', custom_activation) register_custom_layer('custom_layer', CustomLayer, {'output_dim': int_param(1, 100)}) with tempfile.TemporaryDirectory() as tmp_dir: batch_size = 50 batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset( batch_size, 1000) layout = Layout(input_size=1000, output_size=nb_classes, output_activation='softmax') training = Training( objective=Objective('categorical_crossentropy'), optimizer=Optimizer(optimizer='Adam'), metric=Metric('categorical_accuracy'), stopping=EpochStoppingCondition(10), batch_size=batch_size) experiment_parameters = ExperimentParameters( use_default_values=True) experiment_parameters.layout_parameter('rows', 1) experiment_parameters.layout_parameter('blocks', 1) experiment_parameters.layout_parameter('layers', 1) experiment_parameters.layout_parameter('block.layer_type', 'custom_layer') experiment = Experiment('test__reuters_experiment', layout, training, batch_iterator, test_batch_iterator, CpuEnvironment(n_jobs=1, data_dir=tmp_dir), parameters=experiment_parameters) blueprint = create_random_blueprint(experiment) model = ModelBuilder().build(blueprint, default_device()) model.fit_generator( generator=batch_iterator, samples_per_epoch=batch_iterator.samples_per_epoch, nb_epoch=10, validation_data=test_batch_iterator, nb_val_samples=test_batch_iterator.sample_count) filepath = join(tmp_dir, 'model') model.save(filepath) model = load_keras_model(filepath) self.assertIsNotNone(model, 'Should have loaded the model')
def test_train(self): disable_sysout() with tempfile.TemporaryDirectory() as tmp_dir: batch_size = 50 batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset( batch_size, 1000) layout = Layout(input_size=1000, output_size=nb_classes, output_activation='softmax') training = Training( objective=Objective('categorical_crossentropy'), optimizer=Optimizer(optimizer='Adam'), metric=Metric('categorical_accuracy'), stopping=EpochStoppingCondition(10), batch_size=batch_size) experiment_parameters = ExperimentParameters( use_default_values=True) experiment_parameters.layout_parameter('rows', 1) experiment_parameters.layout_parameter('blocks', 1) experiment_parameters.layout_parameter('layers', 1) experiment = Experiment('test__reuters_experiment', layout, training, batch_iterator, test_batch_iterator, CpuEnvironment(n_jobs=1, data_dir=tmp_dir), parameters=experiment_parameters) blueprint = create_random_blueprint(experiment) model = ModelBuilder().build(blueprint, default_device()) result = model.fit_generator( generator=batch_iterator, samples_per_epoch=batch_iterator.samples_per_epoch, nb_epoch=10, validation_data=test_batch_iterator, nb_val_samples=test_batch_iterator.sample_count) self.assertIsNotNone(result, 'should have fit the model') score = model.evaluate_generator( test_batch_iterator, val_samples=test_batch_iterator.sample_count) self.assertIsNotNone(score, 'should have evaluated the model')
def test_mutate_parameters(self): layout = Layout( input_size=100, output_size=10, output_activation='softmax') training = Training( objective=None, optimizer=None, metric=None, stopping=None, batch_size=None) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=ExperimentParameters(use_default_values=False)) experiment.parameters.all_search_parameters(True) for _ in range(10): blueprint = create_random_blueprint(experiment) mutant = mutate_blueprint( blueprint, parameters=experiment.parameters, p_mutate_layout=0, p_mutate_param=1, mutate_in_place=False) for row_idx, row in enumerate(mutant.layout.rows): for block_idx, block in enumerate(row.blocks): for layer_idx, layer in enumerate(block.layers): original_row = blueprint.layout.rows[row_idx] original_block = original_row.blocks[block_idx] original_layer = original_block.layers[layer_idx] for name, value in layer.parameters.items(): if value == original_layer.parameters[name]: pass self.assertTrue( value != original_layer.parameters[name], 'Should have mutated parameter')
def test_model_trainer(self): disable_sysout() with tempfile.TemporaryDirectory() as tmp_dir: batch_size = 50 batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset( batch_size, 1000) layout = Layout(input_size=1000, output_size=nb_classes, output_activation='softmax') training = Training( objective=Objective('categorical_crossentropy'), optimizer=Optimizer(optimizer='Adam'), metric=Metric('categorical_accuracy'), stopping=EpochStoppingCondition(10), batch_size=batch_size) experiment_parameters = ExperimentParameters( use_default_values=True) experiment_parameters.layout_parameter('rows', 1) experiment_parameters.layout_parameter('blocks', 1) experiment_parameters.layout_parameter('layers', 1) experiment = Experiment('test__reuters_experiment', layout, training, batch_iterator, test_batch_iterator, CpuEnvironment(n_jobs=1, data_dir=tmp_dir), parameters=experiment_parameters) blueprint = create_random_blueprint(experiment) trainer = ModelTrainer(batch_iterator, test_batch_iterator) model_filename = join(tmp_dir, 'model') model, history, _duration = trainer.train( blueprint, cpu_device(), save_best_model=True, model_filename=model_filename) self.assertIsNotNone(model, 'should have fit the model') self.assertTrue(isfile(model_filename), 'Should have saved the model') self.assertIsNotNone(history, 'Should have the training history')
def test_build_w_custom_definitions(self): def custom_activation(x): return x register_custom_activation('custom_activation', custom_activation) register_custom_layer( 'Dense2', Dense, deepcopy(reference_parameters['layers']['Dense']), True) layout = Layout(input_size=100, output_size=10, output_activation='softmax', block=['Dense2']) training = Training(objective=Objective('categorical_crossentropy'), optimizer=None, metric=Metric('categorical_accuracy'), stopping=EpochStoppingCondition(5), batch_size=250) experiment_parameters = ExperimentParameters(use_default_values=False) experiment_parameters.layout_parameter('blocks', int_param(1, 5)) experiment_parameters.layout_parameter('layers', int_param(1, 5)) experiment_parameters.layout_parameter('layer.type', string_param(['Dense2'])) experiment_parameters.layer_parameter('Dense2.output_dim', int_param(10, 500)) experiment_parameters.layer_parameter( 'Dense2.activation', string_param(['custom_activation'])) experiment_parameters.layer_parameter('Dropout.p', float_param(0.1, 0.9)) experiment_parameters.all_search_parameters(True) experiment = Experiment('test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=experiment_parameters) check_experiment_parameters(experiment) for _ in range(5): blueprint1 = create_random_blueprint(experiment) for layer in blueprint1.layout.get_layers(): self.assertEqual('Dense2', layer.layer_type, 'Should have used custom layer') model = ModelBuilder().build(blueprint1, cpu_device()) self.assertIsNotNone(model, 'Should have built a model') blueprint2 = create_random_blueprint(experiment) for layer in blueprint2.layout.get_layers(): self.assertEqual('Dense2', layer.layer_type, 'Should have used custom layer') model = ModelBuilder().build(blueprint2, cpu_device()) self.assertIsNotNone(model, 'Should have built a model') blueprint3 = mix_blueprints(blueprint1, blueprint2, experiment_parameters) for layer in blueprint3.layout.get_layers(): self.assertEqual('Dense2', layer.layer_type, 'Should have used custom layer') model = ModelBuilder().build(blueprint3, cpu_device()) self.assertIsNotNone(model, 'Should have built a model') blueprint4 = mutate_blueprint(blueprint1, experiment_parameters, mutate_in_place=False) for layer in blueprint4.layout.get_layers(): self.assertEqual('Dense2', layer.layer_type, 'Should have used custom layer') model = ModelBuilder().build(blueprint4, cpu_device()) self.assertIsNotNone(model, 'Should have built a model')
def test_predefined_layout(self): layout = Layout( input_size=100, output_size=10, output_activation='softmax', block=[ ('Dense', {'activation': 'relu'}), 'Dropout', ('Dense', {'output_dim': 100})]) training = Training( objective=None, optimizer=Optimizer(), metric=None, stopping=None, batch_size=None) experiment = Experiment( 'test', layout, training, batch_iterator=None, test_batch_iterator=None, environment=None, parameters=ExperimentParameters(use_default_values=False)) experiment.parameters.search_parameter('layout', False) experiment.parameters.search_parameter('parameters', True) experiment.parameters.search_parameter('optimizer', True) for _ in range(10): blueprint1 = create_random_blueprint(experiment) blueprint2 = create_random_blueprint(experiment) blueprint3 = mix_blueprints(blueprint1, blueprint2, experiment.parameters) blueprint4 = mutate_blueprint(blueprint1, experiment.parameters, mutate_in_place=False) for idx, blueprint in enumerate([blueprint1, blueprint2, blueprint3, blueprint4]): self.assertIsNotNone(blueprint, 'Should have created a blueprint') self.assertIsNotNone(blueprint.layout, 'Should have created a layout') self.assertEqual( 1, len(blueprint.layout.get_rows()), 'Should have 1 row') self.assertEqual( 1, len(blueprint.layout.get_blocks()), 'Should have 1 block') self.assertEqual( len(layout.block), len(blueprint.layout.get_layers()), 'Should have predefined layers count') for i, row in enumerate(blueprint.layout.get_rows()): blocks = len(row.get_blocks()) self.assertTrue( is_valid_param_value( experiment.parameters.get_layout_parameter('blocks'), blocks), 'Invalid value') for block in row.get_blocks(): self.assertEqual( len(layout.block), len(block.layers), 'Should have used template') for i in range(len(layout.block)): layer = layout.block[i] layer_type = str_param_name(layer[0] if isinstance(layer, tuple) else layer) params = layer[1] if isinstance(layer, tuple) else dict() self.assertEqual( layer_type, block.layers[i].layer_type, 'Should have used the predefined layer type') for name, value in params.items(): self.assertEqual( value, block.layers[i].parameters[name], 'Should have used the predefined parameter value for blueprint %d' % idx)
def random_individual(self): return GaBlueprint(individual=create_random_blueprint(self))