def test_loss_is_equal_when_no_jumps(self): connections = ((-1, 2), (-2, 2), (2, 0), (2, 1)) genome = generate_genome_given_graph(graph=connections, connection_weights=(1.0, 2.0, 3.0, 0.5)) dataset = get_dataset(dataset=self.config.dataset, train_percentage=0.1, testing=False, noise=0.0, random_state=self.config.dataset_random_state) n_samples = 100 stg_trainer = StandardTrainer(dataset=dataset, n_samples=self.config.n_samples, n_output=genome.n_output, problem_type=self.config.problem_type, beta=self.config.beta, n_epochs=100, is_cuda=False) stg_trainer.train(genome) best_network = stg_trainer.get_best_network() new_genome = convert_stochastic_network_to_genome(network=best_network, original_genome=genome, fitness=-stg_trainer.best_loss_val, fix_std=genome.genome_config.fix_std) # evaluate genome loss = get_loss(problem_type=self.config.problem_type) loss_value = evaluate_genome(genome=new_genome, dataset=dataset, loss=loss, problem_type=self.config.problem_type, beta_type=self.config.beta_type, batch_size=self.config.batch_size, n_samples=n_samples, is_gpu=False, is_testing=False, return_all=False, is_pass=True) self.assertAlmostEqual(loss_value, -new_genome.fitness, places=0)
def _show_classification_metrics(self, config): dataset = get_dataset(config.dataset, train_percentage=config.train_percentage, testing=False, random_state=config.dataset_random_state, noise=config.noise, label_noise=config.label_noise) loss = get_loss(problem_type=config.problem_type) x, y_true, y_pred, loss_value = evaluate_genome( genome=self.best_individual, dataset=dataset, loss=loss, problem_type=config.problem_type, beta_type=config.beta_type, batch_size=config.batch_size, n_samples=config.n_samples, is_gpu=config.is_gpu, is_testing=True, return_all=True) y_pred = torch.argmax(y_pred, dim=1) from sklearn.metrics import confusion_matrix, accuracy_score # print(f'Loss: {loss_value}') confusion_m = confusion_matrix(y_true, y_pred) acc = accuracy_score(y_true, y_pred) * 100 self.metrics_best['confusion_matrix'] = confusion_m self.metrics_best['accuracy'] = acc print('Confusion Matrix:') print(confusion_m) print(f'Accuracy: {acc} %')
def _parallelize_with_workers(n_cpus, population): # create workers manager = Manager() task_queue = manager.Queue() exit_queue = manager.Queue() exception_queue = manager.Queue() results_queue = manager.Queue() # task_queue = SimpleQueue() # exit_queue = SimpleQueue() # exception_queue = SimpleQueue() # results_queue = SimpleQueue() workers = [] for i in range(n_cpus): worker = Worker(task_queue=task_queue, exit_queue=exit_queue, exception_queue=exception_queue, results_queue=results_queue) worker.start() workers.append(worker) for genome in population.values(): task_queue.put( Task( genome=genome.copy(), dataset=None, x=torch.zeros((2, 784)).float(), y=torch.zeros(2).short(), # x=self.dataset.x.clone().detach(), # y=self.dataset.y.clone().detach(), loss=get_loss('classification'), beta_type='other', problem_type='classification', batch_size=100000, n_samples=20, is_gpu=False)) # loss=self.loss, # beta_type=self.config.beta_type, problem_type=self.config.problem_type, # batch_size=self.batch_size, n_samples=n_samples, is_gpu=self.is_gpu)) while not task_queue.empty(): print('reading results from workers') print(task_queue.qsize()) # for worker in workers: # print(f'Is alive: {worker.is_alive()}') # print('reading results from workers') # print(task_queue.qsize()) if not exception_queue.empty(): exception = exception_queue.get() raise exception results = results_queue.get() print(results) population[results[0]].fitness = -results[1] # sys.exit() # terminate workers # TODO: workers can live during the whole process for i in range(n_cpus): print('sending exit conditions') exit_queue.put(1)
def main(): ALGORITHM_VERSION = 'bayes-neat' DATASET = 'mnist_binary' CORRELATION_ID = 'test' # execution_id = 'f6d2d5e3-26a3-4069-9071-b74009323761' # 2 hours run execution_id = 'bf516f54-c29b-4f88-949c-102ab67930b3' # 10 hours run (learning architecture) # execution_id = '59cbe09c-4ee7-4e7e-9b17-26c866113cfe' # test-run # execution_id = 'c5551a6c-177b-4c2c-8ecd-a75e79ae0ec2' execution_id = '1f30c172-9056-4012-9651-0765527bd550' # fitness -0.2 execution_id = 'a91761a0-6201-4a1d-9293-5e713f305fbf' # fitness -0.86 execution_id = '991b275d-6282-4f7d-8e97-3908baf94726' report_repository = ReportRepository.create(project='neuro-evolution', logs_path=LOGS_PATH) report = report_repository.get_report(algorithm_version=ALGORITHM_VERSION, dataset=DATASET, correlation_id=CORRELATION_ID, execution_id=execution_id) genome_dict = report.data['best_individual'] best_individual_fitness = report.data['best_individual_fitness'] print(f'Fitness of best individual: {best_individual_fitness}') genome = Genome.create_from_julia_dict(genome_dict) # config = get_configuration() print(f'Execution id: {execution_id}') loss = get_loss(problem_type=config.problem_type) ##### EVALUATE ###### print('Evaluating results') evaluate_with_parallel(genome, loss, config) dataset = get_dataset(config.dataset, testing=True) dataset.generate_data() # TODO: remove data-loader. If we want to sample the dataset in each generation, the we can create a # middlelayer between evaluation and dataset x, y_true, y_pred, loss_value = evaluate_genome( genome=genome, dataset=dataset, loss=loss, problem_type=config.problem_type, beta_type=config.beta_type, batch_size=config.batch_size, n_samples=config.n_samples, is_gpu=config.is_gpu, return_all=True) y_pred = torch.argmax(y_pred, dim=1) from sklearn.metrics import confusion_matrix, accuracy_score print(f'Loss: {loss_value}') print('Confusion Matrix:') print(confusion_matrix(y_true, y_pred)) print(f'Accuracy: {accuracy_score(y_true, y_pred) * 100} %')
def evaluate(self, population: dict): ''' population: is a Dict{Int, Genome} ''' logger.info(f'Population size is {len(population)}') # TODO: make n_samples increase with generation number n_samples = self.config.n_samples if self.parallel_evaluation: tasks = [] for genome in population.values(): # logger.debug(f'Genome {genome.key}: {genome.get_graph()}') x = (genome, get_loss(problem_type=self.config.problem_type), self.config.beta_type, self.config.problem_type, self.config.n_input, self.config.n_output, self.config.node_activation, self.batch_size, n_samples, self.is_gpu) tasks.append(x) # TODO: fix logging when using multiprocessing. Easy fix is to disable fitnesses = list( self.pool.imap(evaluate_genome_task_jupyneat, tasks, chunksize=len(population) // self.n_processes)) for i, genome in enumerate(population.values()): genome['fitness'] = fitnesses[i] else: self.dataset = self._get_dataset() self.loss = self._get_loss() for genome in population.values(): genome['fitness'] = -evaluate_genome_jupyneat( genome=genome, problem_type=self.config.problem_type, n_input=self.config.n_input, n_output=self.config.n_output, activation_type=self.config.node_activation, dataset=self.dataset, loss=self.loss, beta_type=self.config.beta_type, batch_size=self.batch_size, n_samples=n_samples, is_gpu=self.is_gpu) return population
def main(): ALGORITHM_VERSION = 'bayes-neat' DATASET = 'classification_example_1' CORRELATION_ID = 'test' execution_id = '180186eb-46c8-4bbd-9f8a-26a36cbe57e4' report_repository = ReportRepository.create(project='neuro-evolution', logs_path=LOGS_PATH) report = report_repository.get_report(algorithm_version=ALGORITHM_VERSION, dataset=DATASET, correlation_id=CORRELATION_ID, execution_id=execution_id) genome = report.data['best_individual'] best_individual_fitness = report.data['best_individual_fitness'] print(f'Fitness of best individual: {best_individual_fitness}') config_dict = report.config config = jsons.load(config_dict, BaseConfiguration) loss = get_loss(problem_type=config.problem_type) dataset = get_dataset(config.dataset, testing=True) dataset.generate_data() x, y_true, y_pred, loss_value = evaluate_genome_jupyneat( genome=genome, problem_type=config.problem_type, n_input=config.n_input, n_output=config.n_output, activation_type=config.node_activation, dataset=dataset, loss=loss, beta_type=config.beta_type, batch_size=config.batch_size, n_samples=config.n_samples, is_gpu=config.is_gpu, return_all=True) y_pred = torch.argmax(y_pred, dim=1) from sklearn.metrics import confusion_matrix, accuracy_score print(f'Loss: {loss_value}') print('Confusion Matrix:') print(confusion_matrix(y_true, y_pred)) print(f'Accuracy: {accuracy_score(y_true, y_pred) * 100} %')
def _initialize(self): if IS_ALTERNATIVE_NETWORK: Network = ProbabilisticFeedForwardAlternative else: Network = ProbabilisticFeedForward self.network = Network(n_input=self.config.n_input, n_output=self.config.n_output, is_cuda=self.is_cuda, n_neurons_per_layer=self.n_neurons_per_layer, n_hidden_layers=self.n_hidden_layers) self.network.reset_parameters() self.criterion = get_loss(problem_type=self.config.problem_type) self.optimizer = Adam(self.network.parameters(), lr=self.lr, weight_decay=self.weight_decay) if self.is_cuda: self.network.cuda() self.criterion.cuda()
def evaluate(self, population: dict): ''' population: is a Dict{Int, Genome} ''' # TODO: make n_samples increase with generation number n_samples = self.config.n_samples if self.parallel_evaluation: tasks = [] for genome in population.values(): logger.debug(f'Genome {genome.key}: {genome.get_graph()}') x = (genome, get_loss(problem_type=self.config.problem_type), self.config.beta_type, self.config.problem_type, IS_TESTING, self.batch_size, n_samples, self.is_gpu) tasks.append(x) # TODO: fix logging when using multiprocessing. Easy fix is to disable fitnesses = list(self.pool.imap(evaluate_genome_task, tasks, chunksize=len(population)//self.n_processes)) for i, genome in enumerate(population.values()): genome.fitness = fitnesses[i] else: self.dataset = self._get_dataset() self.loss = self._get_loss() for genome in population.values(): logger.debug(f'Genome {genome.key}: {genome.get_graph()}') genome.fitness = - evaluate_genome(genome=genome, problem_type=self.config.problem_type, dataset=self.dataset, loss=self.loss, is_testing=IS_TESTING, beta_type=self.config.beta_type, batch_size=self.batch_size, n_samples=n_samples, is_gpu=self.is_gpu) return population
loss_value = kl_posterior.item() return loss_value tasks = [] pool = Pool(processes=N_PROCESSES, initializer=process_initialization, initargs=(config.dataset, True)) for genome in genomes: logger.debug(f'Genome {genome.key}: {genome.get_graph()}') # x = torch.zeros(2, 784).float() # y = torch.zeros(2).long() # x=dataset.x.clone().detach() # y=dataset.y.clone().detach() loss = get_loss('classification') beta_type = 'other' problem_type = 'classification' batch_size = 100000 n_samples = 20 is_gpu = False x = (genome.copy(), # x, y, loss, beta_type, problem_type, batch_size, n_samples, is_gpu) tasks.append(x) # TODO: fix logging when using multiprocessing. Easy fix is to disable fitnesses = list(pool.imap(evaluate_genome_parallel, tasks, chunksize=max([len(genomes)//N_PROCESSES, 1])))
def _generate_row(self, report, absolute_best=True): execution_id = report.execution_id correlation_id = report.correlation_id if absolute_best: genome_dict = report.data['best_individual'] best_individual_fitness = report.data['best_individual_fitness'] else: genome_dict = report.data['fine_tuning'][ 'best_genome_before_fine_tuning'] best_individual_fitness = report.data['fine_tuning'][ 'best_fitness_before_fine_tuning'] genome = Genome.from_dict(genome_dict) config = genome.genome_config self.configurations[execution_id] = config self.best_genomes[execution_id] = genome self.best_networks[execution_id] = ComplexStochasticNetwork( genome=genome) set_configuration(config) # evaluate genome loss = get_loss(problem_type=config.problem_type) print(f'Train percentage: {config.train_percentage}') print(f'Random state: {config.dataset_random_state}') dataset = get_dataset(config.dataset, train_percentage=config.train_percentage, testing=True, random_state=config.dataset_random_state, noise=config.noise, label_noise=config.label_noise) x, y_true, y_pred_prob, loss_value = evaluate_genome( genome=genome, dataset=dataset, loss=loss, problem_type=config.problem_type, beta_type=config.beta_type, batch_size=config.batch_size, n_samples=self.n_samples, is_gpu=config.is_gpu, is_testing=True, return_all=True) y_pred = torch.argmax(y_pred_prob, dim=1) train_percentage = config.train_percentage noise = config.noise label_noise = config.label_noise duration = report.duration n_parameters = genome.calculate_number_of_parameters() n_nodes = genome.n_bias_parameters // 2 n_connections = genome.n_weight_parameters // 2 n_layers = self._get_number_of_layers(genome) mean_genome_std = get_mean_std(genome) end_condition = report.data['end_condition'] chunk = pd.DataFrame( { 'correlation_id': correlation_id, 'execution_id': execution_id, 'train_percentage': train_percentage, 'noise': noise, 'label_noise': label_noise, 'is_bayesian': False if config.fix_std else True, 'beta': config.beta, 'loss_training': -best_individual_fitness, 'loss_testing': loss_value, 'duration': duration, 'end_condition': end_condition, 'n_parameters': n_parameters, 'n_nodes': n_nodes, 'n_connections': n_connections, 'n_layers': n_layers, 'mean_genome_std': mean_genome_std, }, index=[0]) if config.problem_type == 'classification': chunk['accuracy'] = accuracy_score(y_true, y_pred) * 100 chunk['precision'] = precision_score(y_true, y_pred, average='weighted') chunk['recall'] = recall_score(y_true, y_pred, average='weighted') chunk['f1'] = f1_score(y_true, y_pred, average='weighted') ece, _ = expected_calibration_error( y_true.numpy(), y_pred_prob.numpy(), n_bins=ECE_N_BINS, uniform_binning=UNIFORM_BINNING) chunk['ece'] = ece else: chunk['mse'] = mean_squared_error(y_true, y_pred) chunk['mae'] = mean_absolute_error(y_true, y_pred) return chunk
def _get_loss(self): if self.loss is None: self.loss = get_loss(problem_type=self.config.problem_type) return self.loss
def train(self, genome): kl_qw_pw = compute_kl_qw_pw(genome=genome) # setup network self.network = ComplexStochasticNetwork(genome=genome, is_trainable=True, is_cuda=self.is_cuda) self.criterion = get_loss(problem_type=self.problem_type) if self.is_cuda: self.network.cuda() self.criterion.cuda() self.optimizer = Adam(self.network.parameters(), lr=self.lr, weight_decay=self.weight_decay) x_batch, y_batch = self.dataset.x_train, self.dataset.y_train x_train, x_val, y_train, y_val = self.train_val_split( x_batch, y_batch, problem_type=self.problem_type, val_ratio=0.2) x_train, _ = _prepare_batch_data( x_batch=x_train, y_batch=y_train, problem_type=self.problem_type, is_gpu=False, # this could be removed n_input=genome.n_input, n_output=genome.n_output, n_samples=self.n_samples) x_val, _ = _prepare_batch_data(x_batch=x_val, y_batch=y_val, problem_type=self.problem_type, is_gpu=False, n_input=genome.n_input, n_output=genome.n_output, n_samples=self.n_samples) if self.is_cuda: x_train = x_train.cuda() y_train = y_train.cuda() x_val = x_val.cuda() y_val = y_val.cuda() self.network.train() for epoch in range(self.n_epochs): loss_epoch = self._train_one(x_train, y_train, kl_qw_pw) # if epoch % 10 == 0: _, _, _, loss_val = self._evaluate(x_val, y_val, network=self.network) if loss_val < self.best_loss_val: self.best_loss_val = loss_val self.best_network_state = copy.deepcopy( self.network.state_dict()) self.last_update = epoch if epoch - self.last_update > N_EPOCHS_WITHOUT_IMPROVING: print( f'Breaking training as not improving for {N_EPOCHS_WITHOUT_IMPROVING} epochs' ) break if epoch % 200 == 0: print(f'Epoch = {epoch}. Training Loss: {loss_epoch}. ' f'Best Val. Loss: {self.best_loss_val}') self.network.clear_non_existing_weights( clear_grad=False) # reset non-existing weights self.final_loss = loss_epoch print(f'Final Epoch = {epoch}. Training Error: {self.final_loss}')