def generate_random_dslm_ols_configuration(option=None, init_maximum_layers=5, maximum_iterations=70,
maximum_neuron_connection_weight=0.5, maximum_bias_connection_weight=1.0,
mutation_maximum_new_neurons_per_layer=3,
init_maximum_cnn_neuron_per_layer=5, init_maximum_cnn_layers=5, maximum_learning_step=0.2):
configuration = {}
configuration['stopping_criterion'] = MaxGenerationsCriterion(random.randint(1, maximum_iterations))
configuration['population_size'] = DEFAULT_POPULATION_SIZE
configuration['layers'] = init_maximum_layers
configuration['cnn_layers'] = init_maximum_cnn_layers
configuration['cnn_neurons_per_layer'] = init_maximum_cnn_neuron_per_layer
configuration['learning_step'] = random.uniform(0.1, maximum_learning_step) #'optimized'
configuration['maximum_neuron_connection_weight'] = random.uniform(0.1, maximum_neuron_connection_weight)
configuration['maximum_bias_connection_weight'] = random.uniform(0.1, maximum_bias_connection_weight)
# configuration['mutation_operator'] = Mutation4(maximum_new_neurons_per_layer=mutation_maximum_new_neurons_per_layer,
# maximum_bias_connection_weight=configuration[
# 'maximum_bias_connection_weight'])
configuration['mutation_operator'] = Mutation_CNN_1(maximum_new_cnn_neurons_per_layer=1)
configuration['random_sampling_technique'] = False
configuration['random_weighting_technique'] = False
configuration['protected_ols'] = False
configuration['bootstrap_ols'] = False
configuration['store_ls_history'] = True
return configuration
def generate_random_1_10_slm_ols_configuration(
init_maximum_layers=5,
maximum_iterations=250,
maximum_neuron_connection_weight=0.5,
maximum_bias_connection_weight=1.0,
mutation_maximum_new_neurons_per_layer=3):
#def generate_random_1_10_slm_ols_configuration(init_maximum_layers=5, maximum_iterations=100, maximum_neuron_connection_weight=0.5, maximum_bias_connection_weight=1.0, mutation_maximum_new_neurons_per_layer=3):
configuration = generate_random_slm_ols_configuration(
init_maximum_layers=init_maximum_layers,
maximum_iterations=maximum_iterations,
maximum_neuron_connection_weight=maximum_neuron_connection_weight,
maximum_bias_connection_weight=maximum_bias_connection_weight,
mutation_maximum_new_neurons_per_layer=
mutation_maximum_new_neurons_per_layer)
configuration['population_size'] = 10
#===========================================================================
# configuration['stopping_criterion'] = MaxGenerationsCriterion(10000)
#===========================================================================
configuration['stopping_criterion'] = MaxGenerationsCriterion(
random.randint(250, 1000))
configuration['random_weighting_technique'] = True
configuration['weight_range'] = 1
return configuration
def generate_random_ftne_configuration(maximum_iterations=100, maximum_number_of_layers=5, maximum_neurons_per_layer=5):
configuration = {}
configuration['stopping_criterion'] = MaxGenerationsCriterion(random.randint(1, maximum_iterations))
#===========================================================================
# configuration['stopping_criterion'] = MaxGenerationsCriterion(DEFAULT_NUMBER_OF_ITERATIONS)
#===========================================================================
configuration['population_size'] = DEFAULT_POPULATION_SIZE
# Jan: [create_network_from_topology(topology) for topology in [[1], [2], [2, 2], [3, 3, 3], [5, 5, 5]]]
number_of_layers = random.randint(1, maximum_number_of_layers)
neurons_per_layer = [random.randint(1, maximum_neurons_per_layer) for i in range(number_of_layers)]
configuration['topology'] = create_network_from_topology(neurons_per_layer)
# Jan: [SelectionOperatorTournament(5)]
tournament_size = random.randint(1, configuration['population_size'])
configuration['selection_operator'] = SelectionOperatorTournament(tournament_size)
# Jan: [MutationOperatorGaussian(0.01), MutationOperatorGaussian(0.1)]
standard_deviation = random.uniform(0.001, 5)
configuration['mutation_operator'] = MutationOperatorGaussian(standard_deviation)
# Jan: [CrossoverOperatorArithmetic()]
configuration['crossover_operator'] = CrossoverOperatorArithmetic()
# Jan: [0.25, 0.5]
configuration['mutation_rate'] = random.uniform(0.001, 1)
# Jan: [0.01, 0.1]
configuration['crossover_rate'] = random.uniform(0.001, 1)
return configuration
def test_benchmark_neat(self):
print('Test BenchmarkNEAT()...')
algorithm = BenchmarkNEAT(10, MaxGenerationsCriterion(10), 4, 1, 1,
0.1, 0.1, 0.1, 0.1, 0.1, 0.1)
X = get_input_variables(self.training).as_matrix()
y = get_target_variable(self.training).as_matrix()
log = algorithm.fit(X, y, RootMeanSquaredError, verbose=True)
self.assertTrue(expr=log)
print()
def test_benchmark_slm(self):
print('Test BenchmarkSLM()...')
algorithm = BenchmarkSLM(10, MaxGenerationsCriterion(10), 3, 0.01, 50,
Mutation2())
X = get_input_variables(self.training).as_matrix()
y = get_target_variable(self.training).as_matrix()
log = algorithm.fit(X, y, RootMeanSquaredError, verbose=True)
self.assertTrue(expr=log)
print()
def test_ols(self):
print('OLS tests of fit()...')
algorithm = SemanticLearningMachine(100, MaxGenerationsCriterion(200),
3, 'optimized', 50, Mutation2())
X = get_input_variables(self.training).as_matrix()
y = get_target_variable(self.training).as_matrix()
algorithm.fit(X, y, RootMeanSquaredError, verbose=True)
self.assertTrue(expr=algorithm.champion)
print()
def generate_random_slm_bls_configuration(
option=None,
init_maximum_layers=5,
maximum_iterations=100,
maximum_learning_step=10,
maximum_neuron_connection_weight=0.5,
maximum_bias_connection_weight=1.0,
mutation_maximum_new_neurons_per_layer=3):
configuration = {}
configuration['stopping_criterion'] = MaxGenerationsCriterion(
random.randint(1, maximum_iterations))
#===========================================================================
# configuration['stopping_criterion'] = MaxGenerationsCriterion(DEFAULT_NUMBER_OF_ITERATIONS)
#===========================================================================
configuration['population_size'] = DEFAULT_POPULATION_SIZE
configuration['layers'] = init_maximum_layers
configuration['learning_step'] = random.uniform(0.001,
maximum_learning_step)
configuration['maximum_neuron_connection_weight'] = random.uniform(
0.1, maximum_neuron_connection_weight)
configuration['maximum_bias_connection_weight'] = random.uniform(
0.1, maximum_bias_connection_weight)
configuration['mutation_operator'] = Mutation4(
maximum_new_neurons_per_layer=mutation_maximum_new_neurons_per_layer,
maximum_bias_connection_weight=configuration[
'maximum_bias_connection_weight'])
configuration['random_sampling_technique'] = False
configuration['random_weighting_technique'] = False
configuration['protected_ols'] = False
configuration['bootstrap_ols'] = False
configuration['store_ls_history'] = True
"""
if option == 0: # no RST and no RWT
config['random_sampling_technique'] = False
config['random_weighting_technique'] = False
elif option == 1: # RST
config['random_sampling_technique'] = True
config['random_weighting_technique'] = False
config['subset_ratio'] = random.uniform(0.01, 0.99)
elif option == 2: # RWT
config['random_sampling_technique'] = False
config['random_weighting_technique'] = True
config['weight_range'] = 1
"""
return configuration
def setUp(self):
self.training, self.validation, self.testing = load_samples(
'r_music', 0)
topology = create_network_from_topology([2, 2])
self.sga = SimpleGeneticAlgorithm(100, MaxGenerationsCriterion(200),
topology,
SelectionOperatorTournament(5),
MutationOperatorGaussian(0.05),
CrossoverOperatorArithmetic(), 0.01,
0.5)
def test_benchmark_sga(self):
print('Test BenchmarkSGA()...')
topology = create_network_from_topology([2, 2])
algorithm = BenchmarkSGA(10, MaxGenerationsCriterion(10), topology,
SelectionOperatorTournament(5),
MutationOperatorGaussian(0.1),
CrossoverOperatorArithmetic(), 0.01, 0.25)
X = get_input_variables(self.training).as_matrix()
y = get_target_variable(self.training).as_matrix()
log = algorithm.fit(X, y, RootMeanSquaredError, verbose=True)
self.assertTrue(expr=log)
print()
def test_ols(self):
print('OLS tests of fit()...')
def time_seconds(): return default_timer()
start_time = time_seconds()
algorithm = SemanticLearningMachine(100, MaxGenerationsCriterion(200), 3, 'optimized', 50, Mutation2(), RootMeanSquaredError, True)
X = get_input_variables(self.training).values
y = get_target_variable(self.training).values
start_time = time_seconds()
algorithm.fit(X, y, RootMeanSquaredError, verbose=False)
print("time to train algorithm: ", (time_seconds()-start_time))
self.assertTrue(expr=algorithm.champion)
print()
def test_slm_fls(self):
print("testing fit() for SLM (FLS) ...")
base_learner = SemanticLearningMachine(50, MaxGenerationsCriterion(100), 2, 1, 10, Mutation2())
ensemble_learner = EnsembleRandomIndependentWeighting(base_learner, 100, weight_range=1)
X = get_input_variables(self.training).values
y = get_target_variable(self.training).values
def time_seconds(): return default_timer()
start_time = time_seconds()
ensemble_learner.fit(X, y, RootMeanSquaredError, verbose=False)
print("time to train algorithm: ", (time_seconds()-start_time))
self.assertTrue(expr=ensemble_learner.learners)
print()
def test_slm_ols_wo_edv(self):
print("testing fit() for SLM (OLS) without EDV ...")
base_learner = SemanticLearningMachine(50, MaxGenerationsCriterion(20), 2, 'optimized', 10, Mutation2())
ensemble_learner = EnsembleBoosting(base_learner, 100, meta_learner=median, learning_rate=1)
X = get_input_variables(self.training).values
y = get_target_variable(self.training).values
def time_seconds(): return default_timer()
start_time = time_seconds()
ensemble_learner.fit(X, y, RootMeanSquaredError, verbose=False)
print("time to train algorithm: ", (time_seconds()-start_time))
self.assertTrue(expr=ensemble_learner.learners)
print()
def generate_random_neat_configuration(maximum_iterations=100):
configuration = {}
""" https://neat-python.readthedocs.io/en/latest/config_file.html """
configuration['stopping_criterion'] = MaxGenerationsCriterion(
random.randint(1, maximum_iterations))
#===========================================================================
# configuration['stopping_criterion'] = MaxGenerationsCriterion(DEFAULT_NUMBER_OF_ITERATIONS)
#===========================================================================
configuration['population_size'] = DEFAULT_POPULATION_SIZE
# Jan: [3, 4]
"""" Individuals whose genomic distance is less than this threshold are considered to be in the same species """
configuration['compatibility_threshold'] = random.randint(1, 10)
# Jan: [1]
""" The coefficient for the disjoint and excess gene counts’ contribution to the genomic distance. """
configuration['compatibility_disjoint_coefficient'] = random.uniform(
0.001, 1)
# Jan: [1]
""" The coefficient for each weight, bias, or response multiplier difference’s contribution to the genomic distance (for homologous nodes or connections). This is also used as the value to add for differences in activation functions, aggregation functions, or enabled/disabled status. """
configuration['compatibility_weight_coefficient'] = random.uniform(
0.001, 1)
# Jan: [0.1, 0.25]
""" The probability that mutation will add a connection between existing nodes. Valid values are in [0.0, 1.0]. """
configuration['conn_add_prob'] = random.uniform(0.001, 1)
# Jan: [0.1]
""" The probability that mutation will delete an existing connection. Valid values are in [0.0, 1.0]. """
configuration['conn_delete_prob'] = random.uniform(0.001, 1)
# Jan: [0.1, 0.25]
""" The probability that mutation will add a new node (essentially replacing an existing connection, the enabled status of which will be set to False). Valid values are in [0.0, 1.0]. """
configuration['node_add_prob'] = random.uniform(0.001, 1)
# Jan: [0.1]
""" The probability that mutation will delete an existing node (and all connections to it). Valid values are in [0.0, 1.0]. """
configuration['node_delete_prob'] = random.uniform(0.001, 1)
# Jan: [0.25]
""" The probability that mutation will change the weight of a connection by adding a random value. """
configuration['weight_mutate_rate'] = random.uniform(0.001, 1)
# Jan: [0.25]
""" The standard deviation of the zero-centered normal/gaussian distribution from which a weight value mutation is drawn. """
configuration['weight_mutate_power'] = random.uniform(0.001, 5)
return configuration
def get_random_config_slm_fls_grouped(
option=None,
init_maximum_layers=5,
maximum_iterations=200,
maximum_learning_step=1,
maximum_neuron_connection_weight=0.5,
maximum_bias_connection_weight=1.0,
mutation_maximum_new_neurons_per_layer=3):
config = {}
config['stopping_criterion'] = MaxGenerationsCriterion(
random.randint(1, maximum_iterations))
config['population_size'] = 10
config['layers'] = init_maximum_layers
# config['learning_step'] = random.uniform(0.1, maximum_learning_step)
config['learning_step'] = random.uniform(0.001, maximum_learning_step)
config['maximum_neuron_connection_weight'] = random.uniform(
0.1, maximum_neuron_connection_weight)
config['maximum_bias_connection_weight'] = random.uniform(
0.1, maximum_bias_connection_weight)
config['mutation_operator'] = Mutation4(
maximum_new_neurons_per_layer=mutation_maximum_new_neurons_per_layer,
maximum_bias_connection_weight=config['maximum_bias_connection_weight']
)
config['random_sampling_technique'] = False
config['random_weighting_technique'] = False
"""
if option == 0: # no RST and no RWT
config['random_sampling_technique'] = False
config['random_weighting_technique'] = False
elif option == 1: # RST
config['random_sampling_technique'] = True
config['random_weighting_technique'] = False
config['subset_ratio'] = random.uniform(0.01, 0.99)
elif option == 2: # RWT
config['random_sampling_technique'] = False
config['random_weighting_technique'] = True
config['weight_range'] = 1
"""
return config
def setUp(self):
self.training, self.validation, self.testing = load_samples('c_cancer', 0)
self.neat = Neat(100, MaxGenerationsCriterion(200),
4, 1, 1, 0.1, 0.1, 0.1, 0.1, 0.5, 0.01)
from algorithms.common.stopping_criterion import MaxGenerationsCriterion, ErrorDeviationVariationCriterion
from algorithms.common.neural_network.neural_network import create_network_from_topology
from algorithms.semantic_learning_machine.mutation_operator import Mutation2, Mutation3, Mutation4
from algorithms.simple_genetic_algorithm.selection_operator import SelectionOperatorTournament
from algorithms.simple_genetic_algorithm.mutation_operator import MutationOperatorGaussian
from algorithms.simple_genetic_algorithm.crossover_operator import CrossoverOperatorArithmetic
from algorithms.semantic_learning_machine.algorithm import SemanticLearningMachine
from itertools import product
from numpy import mean
_BASE_PARAMETERS = {'number_generations': 200, 'population_size': 100}
_SLM_FLS_PARAMETERS = {
'stopping_criterion':
[MaxGenerationsCriterion(_BASE_PARAMETERS.get('number_generations'))],
'population_size': [_BASE_PARAMETERS.get('population_size')],
'layers': [1, 2, 3],
'learning_step': [0.01],
'max_connections': [1, 10, 50],
'mutation_operator': [Mutation2()]
}
_SLM_OLS_PARAMETERS = {
'stopping_criterion': [
ErrorDeviationVariationCriterion(0.25),
ErrorDeviationVariationCriterion(0.5)
],
'population_size': [_BASE_PARAMETERS.get('population_size')],
'layers': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
'learning_step': ['optimized'],
'max_connections': [1, 10, 50, 100],
def run_nested_cv(self):
""" runs benchmark study on a nested cross-validation environment """
#=======================================================================
# print('self.learning_metric =', self.learning_metric)
# print('self.selection_metric =', self.selection_metric)
#=======================================================================
""" N configuration for each method, trained on all data, selected from the same data """
print('Entering training benchmark for dataset:', self.dataset_name)
training_data = pd.DataFrame(self.samples.values)
for outer_cv in range(_OUTER_FOLDS):
print('\n\tIndex of outer fold:', outer_cv)
for key in self.models.keys():
print('\t\tAlgorithm with key:', key)
if not self.results[key][outer_cv]:
if self.classification:
best_training_value = float('-Inf')
else:
best_training_value = float('-Inf') if self.selection_metric.greater_is_better else float('Inf')
training_value_list = list()
for configuration in range(self.models[key]['max_combinations']):
print('\n\t\t\tIndex of algorithm configuration:', len(training_value_list))
if(len(self.models[key]['algorithms'])) > 1:
option = randint(0, 2)
algorithm = self.models[key]['algorithms'][option]
config = self.models[key]['configuration_method'](option)
else:
algorithm = self.models[key]['algorithms'][0]
#===================================================
# if (key == 'mlpc_sgd' or key == 'mlpc_adam' or key == 'mlpr_sgd' or key == 'mlpr_adam'):
#===================================================
if key.startswith('mlp'):
# version from 01-22
# config = self.models[key]['configuration_method'](self.get_dataset_size(training_outer))
# version from 01-25
batch_size = int(training_data.shape[0])
# batch_size = int(training_outer.shape[0] / _INNER_FOLDS) * 2
config = self.models[key]['configuration_method'](batch_size)
else:
config = self.models[key]['configuration_method']()
if key.startswith('mlp'):
config['max_iter'] = DEFAULT_NUMBER_OF_ITERATIONS
else:
config['stopping_criterion'] = MaxGenerationsCriterion(DEFAULT_NUMBER_OF_ITERATIONS)
results = self._evaluate_algorithm(algorithm=algorithm, configurations=config,
training_set=training_data, validation_set=None, testing_set=training_data, metric=self.learning_metric)
if self.classification:
training_value = results['training_accuracy']
else:
training_value = results['training_value']
if self.classification:
print("\t\t\tAUROC training: %.3f" % (training_value))
else:
print("\t\t\tRMSE training: %.3f" % (training_value))
if self.classification:
if training_value > best_training_value:
#===============================================
# print('\n\t\t\t\t\tClassification: %.3f is better than %.3f\n' % (training_value, best_training_value))
#===============================================
best_algorithm = algorithm
best_key = key
best_configuration = config
best_training_value = training_value
self.results[key][outer_cv] = results
self.results[key][outer_cv]['best_configuration'] = best_configuration
self.results[key][outer_cv]['avg_inner_validation_error'] = best_training_value
self.results[key][outer_cv]['avg_inner_training_error'] = best_training_value
best_overall_algorithm = best_algorithm
best_overall_configuration = best_configuration
best_overall_key = best_key
self.best_result[outer_cv] = self.results[key][outer_cv]
self.best_result[outer_cv]['best_overall_algorithm'] = best_overall_algorithm
self.best_result[outer_cv]['best_overall_configuration'] = best_overall_configuration
self.best_result[outer_cv]['best_overall_key'] = best_overall_key
#===================================================
# else:
# print('\n\t\t\t\t\tClassification: %.3f is worse (!) than %.3f\n' % (training_value, best_training_value))
#===================================================
else:
if is_better(training_value, best_training_value, self.selection_metric):
#===============================================
# print('\n\t\t\t\t\tRegression: %.3f is better than %.3f\n' % (training_value, best_training_value))
#===============================================
best_algorithm = algorithm
best_key = key
best_configuration = config
best_training_value = training_value
self.results[key][outer_cv] = results
self.results[key][outer_cv]['best_configuration'] = best_configuration
self.results[key][outer_cv]['avg_inner_validation_error'] = best_training_value
self.results[key][outer_cv]['avg_inner_training_error'] = best_training_value
best_overall_algorithm = best_algorithm
best_overall_configuration = best_configuration
best_overall_key = best_key
self.best_result[outer_cv] = self.results[key][outer_cv]
self.best_result[outer_cv]['best_overall_algorithm'] = best_overall_algorithm
self.best_result[outer_cv]['best_overall_configuration'] = best_overall_configuration
self.best_result[outer_cv]['best_overall_key'] = best_overall_key
#===================================================
# else:
# print('\n\t\t\t\t\tRegression: %.3f is worse (!) than %.3f\n' % (training_value, best_training_value))
#===================================================
training_value_list.append((configuration, training_value))
if self.classification:
print("\n\t\tAUROC training: %.3f" % (self.results[key][outer_cv]['training_accuracy']))
else:
print("\n\t\tRMSE training: %.3f" % (self.results[key][outer_cv]['training_value']))
# Serialize benchmark
benchmark_to_pickle(self)
print('Leaving training benchmark for dataset:', self.dataset_name)
def setUp(self):
base_learner = SemanticLearningMachine(50, MaxGenerationsCriterion(10), 2, 'optimized', 10, Mutation2())
self.ensemble_learner = Ensemble(base_learner, 50)
self.training, self.validation, self.testing = load_samples('c_diabetes', 0)
