def one_strategy(self, weighted_population):
self.current_chosen_population_sample = [self.subject_id]
for i, pop in enumerate(weighted_population):
start_time = time.time()
if NAS_utils.check_age(pop):
weighted_population[i] = weighted_population[i - 1]
weighted_population[i]['train_time'] = 0
weighted_population[i]['num_epochs'] = 0
continue
finalized_model = finalize_model(pop['model'])
self.current_model_index = i
final_time, evaluations, model, model_state, num_epochs = \
self.activate_model_evaluation(finalized_model, pop['model_state'], subject=self.subject_id)
if global_vars.get('grid_as_ensemble') and global_vars.get(
'delete_finalized_models'):
pop['weighted_avg_params'] = model
self.current_model_index = -1
NAS_utils.add_evaluations_to_weighted_population(
weighted_population[i], evaluations)
weighted_population[i]['model_state'] = model_state
weighted_population[i]['train_time'] = final_time
weighted_population[i]['finalized_model'] = model
weighted_population[i]['num_epochs'] = num_epochs
end_time = time.time()
show_progress(end_time - start_time, self.exp_name)
print('trained model %d in generation %d' %
(i + 1, self.current_generation))
def test_perm_ensemble_fitness(self):
global_vars.set('pop_size', 10)
global_vars.set('ensemble_size', 2)
global_vars.set('ga_objective', 'accuracy')
global_vars.set('permanent_ensembles', True)
dummy_weighted_pop = [{
'val_raw': [[1 - (1 / i), 0, 0, 1 / i]],
'val_target': [3]
} for i in range(1, 11)]
old_len = len(dummy_weighted_pop)
NAS_utils.permanent_ensemble_fitness(dummy_weighted_pop)
NAS_utils.sort_population(dummy_weighted_pop)
assert len(dummy_weighted_pop) == old_len
print(dummy_weighted_pop[-1])
def breed_perm_ensembles(weighted_population, breeding_method, eegnas):
children = []
ensembles = list(
NAS_utils.chunks(list(range(len(weighted_population))),
global_vars.get('ensemble_size')))
while len(weighted_population) + len(children) < global_vars.get(
'pop_size'):
breeders = random.sample(ensembles, 2)
first_ensemble = [weighted_population[i] for i in breeders[0]]
second_ensemble = [weighted_population[i] for i in breeders[1]]
for ensemble in [first_ensemble, second_ensemble]:
assert (len(
np.unique([pop['perm_ensemble_id'] for pop in ensemble])) == 1)
first_ensemble_states = [
NAS_utils.get_model_state(pop) for pop in first_ensemble
]
second_ensemble_states = [
NAS_utils.get_model_state(pop) for pop in second_ensemble
]
new_ensemble, new_ensemble_states, cut_point = breed_two_ensembles(
breeding_method,
mutation_rate=self.mutation_rate,
first_ensemble=first_ensemble,
second_ensemble=second_ensemble,
first_ensemble_states=first_ensemble_states,
second_ensemble_states=second_ensemble_states)
if None not in new_ensemble:
for new_model, new_model_state in zip(new_ensemble,
new_ensemble_states):
children.append({
'model':
new_model,
'model_state':
new_model_state,
'age':
0,
'first_parent_index':
first_ensemble[0]['perm_ensemble_id'],
'second_parent_index':
second_ensemble[0]['perm_ensemble_id'],
'parents': [first_ensemble[0], second_ensemble[0]],
'cut_point':
cut_point
})
EEGNAS.utilities.NAS_utils.hash_model(new_model,
eegnas.models_set,
eegnas.genome_set)
weighted_population.extend(children)
def selection_normal(self, weighted_population):
for index, model in enumerate(weighted_population):
decay_functions = {
'linear': lambda x: x,
'log': lambda x: np.sqrt(np.log(x + 1))
}
if random.uniform(
0, 1) < decay_functions[global_vars.get('decay_function')](
index / global_vars.get('pop_size')):
NAS_utils.remove_from_models_hash(model['model'],
self.models_set,
self.genome_set)
del weighted_population[index]
else:
model['age'] += 1
return weighted_population
def evaluate_and_sort(self, weighted_population):
self.evo_strategy(weighted_population)
getattr(EEGNAS.evolution.fitness_functions,
global_vars.get('fitness_function'))(weighted_population)
if global_vars.get('fitness_penalty_function'):
getattr(NAS_utils, global_vars.get('fitness_penalty_function'))(
weighted_population)
reverse_order = True
if self.loss_function == F.mse_loss:
reverse_order = False
weighted_population = NAS_utils.sort_population(weighted_population,
reverse=reverse_order)
stats = self.calculate_stats(weighted_population)
add_parent_child_relations(weighted_population, stats)
if global_vars.get('ranking_correlation_num_iterations'):
NAS_utils.ranking_correlations(weighted_population, stats)
return stats, weighted_population
def evaluate_ind_deap(self, individual):
try:
finalized_model = finalize_model(individual['model'])
final_time, evaluations, model, model_state, num_epochs = \
self.activate_model_evaluation(finalized_model, individual['model_state'], subject=self.subject_id)
NAS_utils.add_evaluations_to_weighted_population(
individual, evaluations)
individual['model_state'] = model_state
individual['train_time'] = final_time
individual['finalized_model'] = model
individual['num_epochs'] = num_epochs
individual['fitness'] = evaluations[global_vars.get(
'ga_objective')]['valid']
show_progress(final_time, self.exp_name)
return evaluations[global_vars.get('ga_objective')]['valid'],
except Exception as e:
print
def evolution(self):
num_generations = global_vars.get('num_generations')
weighted_population = NAS_utils.initialize_population(
self.models_set, self.genome_set, self.subject_id)
if global_vars.get('puzzle_algo'):
bb_population = [{
'bb': b,
'fitness': 0
} for b in NAS_utils.initialize_bb_population(weighted_population)]
else:
bb_population = None
all_architectures = []
start_time = time.time()
for generation in range(num_generations):
self.current_generation = generation
if global_vars.get('perm_ensembles'):
self.mark_perm_ensembles(weighted_population)
if global_vars.get('inject_dropout') and generation == int(
(num_generations / 2) - 1):
NAS_utils.inject_dropout(weighted_population)
stats, weighted_population = self.evaluate_and_sort(
weighted_population)
if global_vars.get('puzzle_algo'):
NAS_utils.rank_bbs_by_weighted_population(
bb_population, weighted_population)
for stat, val in stats.items():
global_vars.get('sacred_ex').log_scalar(
f'avg_{stat}', val, generation)
if generation < num_generations - 1:
weighted_population = self.selection(weighted_population)
breed_population(weighted_population, self, bb_population)
if bb_population is not None:
NAS_utils.maintain_bb_population(bb_population,
weighted_population)
self.update_mutation_rate()
if global_vars.get('save_every_generation'):
self.save_best_model(weighted_population)
all_architectures.append(
[pop['model'] for pop in weighted_population])
if time.time() - start_time > global_vars.get(
'time_limit_seconds'):
break
pickle.dump(
weighted_population,
open(f'{self.exp_folder}/{self.exp_name}_architectures.p',
'wb'))
self.write_to_csv({k: str(v)
for k, v in stats.items()}, generation + 1)
self.print_to_evolution_file(weighted_population, generation + 1)
best_model_filename = self.save_best_model(weighted_population)
pickle.dump(
weighted_population,
open(f'{self.exp_folder}/{self.exp_name}_architectures.p', 'wb'))
return best_model_filename
def selection_perm_ensembles(self, weighted_population):
ensembles = list(
NAS_utils.chunks(list(range(global_vars.get('pop_size'))),
global_vars.get('ensemble_size')))
for index, ensemble in enumerate(ensembles):
decay_functions = {
'linear': lambda x: x,
'log': lambda x: np.sqrt(np.log(x + 1))
}
if random.uniform(0, 1) < decay_functions[global_vars.get('decay_function')](index / len(ensembles)) and\
len([pop for pop in weighted_population if pop is not None]) > 2 * global_vars.get('ensemble_size'):
for pop in ensemble:
NAS_utils.remove_from_models_hash(
weighted_population[pop]['model'], self.models_set,
self.genome_set)
weighted_population[pop] = None
else:
for pop in ensemble:
weighted_population[pop]['age'] += 1
return [pop for pop in weighted_population if pop is not None]
def test_remove_from_hash(self):
model1 = [
ConvLayer(kernel_eeg_chan=1),
ConvLayer(kernel_eeg_chan=2),
ActivationLayer()
]
model2 = [
ConvLayer(kernel_eeg_chan=2),
ConvLayer(kernel_eeg_chan=1),
ActivationLayer()
]
model_set = []
genome_set = []
EEGNAS.utilities.NAS_utils.hash_model(model1, model_set, genome_set)
EEGNAS.utilities.NAS_utils.hash_model(model2, model_set, genome_set)
assert (len(genome_set)) <= 5
NAS_utils.remove_from_models_hash(model1, model_set, genome_set)
assert (len(genome_set)) >= 3
NAS_utils.remove_from_models_hash(model2, model_set, genome_set)
assert (len(genome_set)) == 0
def run_target_model(self):
global_vars.set('max_epochs', global_vars.get('final_max_epochs'))
global_vars.set('max_increase_epochs',
global_vars.get('final_max_increase_epochs'))
stats = {}
if self.model_from_file is not None:
if type(self.model_from_file) == list:
model = [torch.load(f) for f in self.model_from_file]
else:
model = torch.load(self.model_from_file)
else:
model = target_model(global_vars.get('model_name'))
if global_vars.get('target_pretrain'):
self.datasets['train']['pretrain'], self.datasets['valid']['pretrain'], self.datasets['test']['pretrain'] = \
get_pure_cross_subject(global_vars.get('data_folder'))
nn_trainer = NN_Trainer(self.iterator, self.loss_function,
self.stop_criterion, self.monitors)
dataset = self.get_single_subj_dataset('pretrain',
final_evaluation=False)
_, _, model, _, _ = nn_trainer.train_and_evaluate_model(
model, dataset)
dataset = self.get_single_subj_dataset(self.subject_id,
final_evaluation=True)
nn_trainer = NN_Trainer(self.iterator, self.loss_function,
self.stop_criterion, self.monitors)
_, _, model, _, _ = nn_trainer.train_and_evaluate_model(
model, dataset, final_evaluation=True)
final_time, evaluations, model, model_state, num_epochs =\
nn_trainer.train_and_evaluate_model(model, dataset)
stats['final_train_time'] = str(final_time)
NAS_utils.add_evaluations_to_stats(stats,
evaluations,
str_prefix="final_")
if self.model_from_file is not None:
if type(self.model_from_file) == list:
model_name = f'ensemble_top_{global_vars.get("ensemble_size")}'
else:
model_name = re.findall(r'\d+', self.model_from_file)[-1]
else:
model_name = global_vars.get('model_name')
self.write_to_csv(stats, generation='final', model=model_name)
def breed_normal_population(weighted_population, breeding_method, eegnas,
bb_population):
children = []
while len(weighted_population) + len(children) < global_vars.get(
'pop_size'):
breeders = random.sample(range(len(weighted_population)), 2)
first_breeder = weighted_population[breeders[0]]
second_breeder = weighted_population[breeders[1]]
first_model_state = NAS_utils.get_model_state(first_breeder)
second_model_state = NAS_utils.get_model_state(second_breeder)
new_model, new_model_state, cut_point = breeding_method(
mutation_rate=eegnas.mutation_rate,
first_individual=first_breeder,
second_individual=second_breeder,
first_model_state=first_model_state,
second_model_state=second_model_state,
bb_population=bb_population)
if new_model is not None:
children.append({
'model':
new_model,
'model_state':
new_model_state,
'age':
0,
'parents': [first_breeder, second_breeder],
'cut_point':
cut_point,
'first_parent_index':
breeders[0],
'second_parent_index':
breeders[1],
'weight_inheritance_alpha':
first_breeder['weight_inheritance_alpha'] * 0.5 +
second_breeder['weight_inheritance_alpha'] * 0.5
})
EEGNAS.utilities.NAS_utils.hash_model(new_model, eegnas.models_set,
eegnas.genome_set)
weighted_population.extend(children)
def add_final_stats(self, stats, weighted_population):
model = finalize_model(weighted_population[0]['model'])
if global_vars.get('cross_subject'):
self.current_chosen_population_sample = range(
1,
global_vars.get('num_subjects') + 1)
for subject in self.current_chosen_population_sample:
if global_vars.get('ensemble_iterations'):
ensemble = [
finalize_model(weighted_population[i]['model'])
for i in range(global_vars.get('ensemble_size'))
]
_, evaluations, _, num_epochs = self.ensemble_evaluate_model(
ensemble, final_evaluation=True, subject=subject)
NAS_utils.add_evaluations_to_stats(
stats, evaluations, str_prefix=f"{subject}_final_")
_, evaluations, _, _, num_epochs = self.activate_model_evaluation(
model, final_evaluation=True, subject=subject)
NAS_utils.add_evaluations_to_stats(stats,
evaluations,
str_prefix=f"{subject}_final_")
stats['%d_final_epoch_num' % subject] = num_epochs
def calculate_stats(self, weighted_population):
stats = {}
params = ['train_time', 'num_epochs', 'fitness']
params.extend(NAS_utils.get_metric_strs())
for param in params:
stats[param] = np.mean(
[sample[param] for sample in weighted_population])
if self.subject_id == 'all':
if global_vars.get('cross_subject_sampling_method') == 'model':
self.current_chosen_population_sample = range(
1,
global_vars.get('num_subjects') + 1)
for subject in self.current_chosen_population_sample:
for param in params:
stats[f'{subject}_{param}'] = np.mean([
sample[f'{subject}_{param}']
for sample in weighted_population
if f'{subject}_{param}' in sample.keys()
])
for i, pop in enumerate(weighted_population):
model_stats = {}
for param in params:
model_stats[param] = pop[param]
if self.subject_id == 'all':
if global_vars.get('cross_subject_sampling_method') == 'model':
self.current_chosen_population_sample = range(
1,
global_vars.get('num_subjects') + 1)
for subject in self.current_chosen_population_sample:
for param in params:
if f'{subject}_{param}' in pop.keys():
model_stats[f'{subject}_{param}'] = pop[
f'{subject}_{param}']
if 'parents' in pop.keys():
model_stats['first_parent_child_ratio'] = pop['fitness'] / pop[
'parents'][0]['fitness']
model_stats['second_parent_child_ratio'] = pop[
'fitness'] / pop['parents'][1]['fitness']
model_stats['cut_point'] = pop['cut_point']
model_stats['first_parent_index'] = pop['first_parent_index']
model_stats['second_parent_index'] = pop['second_parent_index']
if global_vars.get('weight_inheritance_alpha') == 'model':
model_stats['weight_inheritance_alpha'] = pop[
'weight_inheritance_alpha']
elif global_vars.get('weight_inheritance_alpha') == 'layer':
for idx in range(len(pop['weight_inheritance_alpha'])):
model_stats[f'weight_inheritance_alpha_{idx}'] = pop[
'weight_inheritance_alpha'][idx]
NAS_utils.add_model_to_stats(pop, i, model_stats)
if i == 0 or i == len(weighted_population) - 1:
for stat, val in model_stats.items():
if 'layer' not in stat:
global_vars.get('sacred_ex').log_scalar(
f'model_{i}_{stat}', val, self.current_generation)
self.write_to_csv(model_stats,
self.current_generation + 1,
model=i)
stats['unique_individuals'] = len(self.models_set)
stats['unique_layers'] = len(self.genome_set)
stats['average_age'] = np.mean(
[sample['age'] for sample in weighted_population])
stats['mutation_rate'] = self.mutation_rate
stats['weight_inheritance_alpha'] = np.mean([
sample['weight_inheritance_alpha']
for sample in weighted_population
])
for layer_type in [
DropoutLayer, ActivationLayer, ConvLayer, IdentityLayer,
BatchNormLayer, PoolingLayer
]:
stats['%s_count' % layer_type.__name__] = \
NAS_utils.count_layer_type_in_pop([pop['model'] for pop in weighted_population], layer_type)
if global_vars.get('add_top_20_stats'):
stats['top20_%s_count' % layer_type.__name__] = \
NAS_utils.count_layer_type_in_pop([pop['model'] for pop in
weighted_population[:int(len(weighted_population)/5)]], layer_type)
if global_vars.get('grid') and not global_vars.get('grid_as_ensemble'):
stats[
'num_of_models_with_skip'] = NAS_utils.num_of_models_with_skip_connection(
weighted_population)
return stats