def mutate_mixture_weights_with_score(self, input_data): if self.score_calc is not None: self._logger.info('Calculating FID/inception score.') # Not necessary for single-cell grids, as mixture must always be [1] if self.neighbourhood.grid_size == 1: best_generators = self.neighbourhood.best_generators dataset = MixedGeneratorDataset( best_generators, self.neighbourhood.mixture_weights_generators, self.score_sample_size, self.cc.settings['trainer'] ['mixture_generator_samples_mode']) self.score = self.score_calc.calculate(dataset)[0] else: # Mutate mixture weights z = np.random.normal( loc=0, scale=self.mixture_sigma, size=len(self.neighbourhood.mixture_weights_generators)) transformed = np.asarray([ value for _, value in self.neighbourhood.mixture_weights_generators.items() ]) transformed += z # Don't allow negative values, normalize to sum of 1.0 transformed = np.clip(transformed, 0, None) transformed /= np.sum(transformed) new_mixture_weights_generators = OrderedDict( zip(self.neighbourhood.mixture_weights_generators.keys(), transformed)) best_generators = self.neighbourhood.best_generators dataset_before_mutation = MixedGeneratorDataset( best_generators, self.neighbourhood.mixture_weights_generators, self.score_sample_size, self.cc.settings['trainer'] ['mixture_generator_samples_mode']) score_before_mutation = self.score_calc.calculate( dataset_before_mutation)[0] del dataset_before_mutation dataset_after_mutation = MixedGeneratorDataset( best_generators, new_mixture_weights_generators, self.score_sample_size, self.cc.settings['trainer'] ['mixture_generator_samples_mode']) score_after_mutation = self.score_calc.calculate( dataset_after_mutation)[0] del dataset_after_mutation # For fid the lower the better, for inception_score, the higher the better if (score_after_mutation < score_before_mutation and self.score_calc.is_reversed) \ or (score_after_mutation > score_before_mutation and (not self.score_calc.is_reversed)): # Adopt the mutated mixture_weights only if the performance after mutation is better self.neighbourhood.mixture_weights_generators = new_mixture_weights_generators self.score = score_after_mutation else: # Do not adopt the mutated mixture_weights here self.score = score_before_mutation
def generate_samples(args, cc): print("generating samples") batch_size = 100 mixture_source = args.mixture_source output_dir = args.output_dir sample_size = args.sample_size dataloader = cc.create_instance(cc.settings['dataloader']['dataset_name']) network_factory = cc.create_instance(cc.settings['network']['name'], dataloader.n_input_neurons) population = Population(individuals=[], default_fitness=0) mixture_definition = read_settings( os.path.join(mixture_source, 'mixture.yml')) for source, weight in mixture_definition.items(): path = os.path.join(mixture_source, source) generator = network_factory.create_generator() generator.net.load_state_dict(torch.load(path)) generator.net.eval() population.individuals.append( Individual(genome=generator, fitness=0, source=source)) dataset = MixedGeneratorDataset( population, mixture_definition, sample_size * batch_size, cc.settings['trainer']['mixture_generator_samples_mode']) os.makedirs(output_dir, exist_ok=True) LipizzanerMaster().save_samples(dataset, output_dir, dataloader, sample_size, batch_size)
def calc_score(args, cc): score_calc = ScoreCalculatorFactory.create() cc.settings['general']['distribution']['client_id'] = 0 dataloader = cc.create_instance(cc.settings['dataloader']['dataset_name']) network_factory = cc.create_instance(cc.settings['network']['name'], dataloader.n_input_neurons) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') generator = network_factory.create_generator() generator.net.load_state_dict( torch.load(args.generator_file, map_location=device)) generator.net.eval() individual = Individual(genome=generator, fitness=0, source='local') dataset = MixedGeneratorDataset( Population(individuals=[individual], default_fitness=0), {'local': 1.0}, 50000, cc.settings['trainer']['mixture_generator_samples_mode']) output_dir = os.path.join(cc.output_dir, 'score') os.makedirs(output_dir, exist_ok=True) LipizzanerMaster().save_samples(dataset, output_dir, dataloader) inc = score_calc.calculate(dataset) _logger.info('Generator loaded from \'{}\' yielded a score of {}'.format( args.generator_file, inc))
def calculate_score(self): best_generators = self.neighbourhood.best_generators dataset = MixedGeneratorDataset( best_generators, self.neighbourhood.mixture_weights_generators, self.score_sample_size) self.score = self.score_calc.calculate(dataset)[0]
def compute_mixture_generative_score(self, epoch=None): # Not necessary for single-cell grids, as mixture must always be [1] self._logger.info("Calculating score {}.".format(epoch)) best_generators = self.neighbourhood.best_generators if True or self.neighbourhood.grid_size == 1: if self.score_calc is not None: dataset = MixedGeneratorDataset( best_generators, self.neighbourhood.mixture_weights_generators, self.score_sample_size, self.cc.settings["trainer"] ["mixture_generator_samples_mode"], # epoch=epoch ) self.score = self.score_calc.calculate(dataset)[0]
def calc_inception_score(args, cc): inception_cal = InceptionCalculator(cuda=True) dataloader = cc.create_instance(cc.settings['dataloader']['dataset_name']) network_factory = cc.create_instance(cc.settings['network']['name'], dataloader.n_input_neurons) generator = network_factory.create_generator() generator.net.load_state_dict(torch.load(args.inception_file)) generator.net.eval() individual = Individual(genome=generator, fitness=0, source='local') dataset = MixedGeneratorDataset( Population(individuals=[individual], default_fitness=0), {'local': 1.0}, 50000) output_dir = os.path.join(cc.output_dir, 'inception_score') os.makedirs(output_dir, exist_ok=True) LipizzanerMaster().save_samples(dataset, output_dir, dataloader) inc = inception_cal.calculate(dataset) _logger.info( 'Generator loaded from \'{}\' yielded an inception score of {}'.format( args.inception_file, inc))
def evaluate_ensemble(self, individual, network_factory, mixture_generator_samples_mode='exact_proportion', fitness_type='tvd'): """It evaluates the solution/individual (ensemble) given the fitness type. It generates samples and it evaluates the metric defined by fitness_type using Lipizzaner. :parameter individual: Solutionto be evaluated :parameter network_factory: :parameter mixture_generator_samples_mode: :parameter fitness_type: It defines the type of metric to be evaluated. :return: The fitness_type metric value got by the solution. """ population = Population(individuals=[], default_fitness=0) # weight_and_generator_indices = [math.modf(gen) for gen in individual] # generators_paths, sources = self.ga.get_generators_for_ensemble(weight_and_generator_indices) # tentative_weights = [weight for weight, generator_index in weight_and_generator_indices] tentative_weights, generators_paths, sources = self.ga.get_mixture_from_individual( individual) mixture_definition = dict(zip(sources, tentative_weights)) for path, source in zip(generators_paths, sources): generator = network_factory.create_generator() generator.net.load_state_dict(torch.load(path, map_location='cpu')) generator.net.eval() population.individuals.append( Individual(genome=generator, fitness=0, source=source)) dataset = MixedGeneratorDataset(population, mixture_definition, 50000, mixture_generator_samples_mode) fid, tvd = self.score_calc.calculate(dataset) if fitness_type == 'tvd': return tvd, elif fitness_type == 'fid': return fid, elif fitness_type == 'tvd-fid': return (tvd, fid),
def _gather_results(self): self._logger.info('Collecting results from clients...') # Initialize node client dataloader = self.cc.create_instance( self.cc.settings['dataloader']['dataset_name']) network_factory = self.cc.create_instance( self.cc.settings['network']['name'], dataloader.n_input_neurons) node_client = NodeClient(network_factory) db_logger = DbLogger() results = node_client.gather_results( self.cc.settings['general']['distribution']['client_nodes'], 120) scores = [] for (node, generator_pop, discriminator_pop, weights_generator, weights_discriminator) in results: node_name = '{}:{}'.format(node['address'], node['port']) try: output_dir = self.get_and_create_output_dir(node) for generator in generator_pop.individuals: source = generator.source.replace(':', '-') filename = '{}{}.pkl'.format(GENERATOR_PREFIX, source) torch.save( generator.genome.net.state_dict(), os.path.join(output_dir, 'generator-{}.pkl'.format(source))) with open(os.path.join(output_dir, 'mixture.yml'), "a") as file: file.write('{}: {}\n'.format( filename, weights_generator[generator.source])) for discriminator in discriminator_pop.individuals: source = discriminator.source.replace(':', '-') filename = '{}{}.pkl'.format(DISCRIMINATOR_PREFIX, source) torch.save(discriminator.genome.net.state_dict(), os.path.join(output_dir, filename)) # Save images dataset = MixedGeneratorDataset( generator_pop, weights_generator, self.cc.settings['master']['score_sample_size'], self.cc.settings['trainer'] ['mixture_generator_samples_mode']) image_paths = self.save_samples(dataset, output_dir, dataloader) self._logger.info( 'Saved mixture result images of client {} to target directory {}.' .format(node_name, output_dir)) # Calculate inception or FID score score = float('-inf') if self.cc.settings['master']['calculate_score']: calc = ScoreCalculatorFactory.create() self._logger.info('Score calculator: {}'.format( type(calc).__name__)) self._logger.info( 'Calculating score score of {}. Depending on the type, this may take very long.' .format(node_name)) score = calc.calculate(dataset) self._logger.info( 'Node {} with weights {} yielded a score of {}'.format( node_name, weights_generator, score)) scores.append((node, score)) if db_logger.is_enabled and self.experiment_id is not None: db_logger.add_experiment_results(self.experiment_id, node_name, image_paths, score) except Exception as ex: self._logger.error( 'An error occured while trying to gather results from {}: {}' .format(node_name, ex)) traceback.print_exc() if self.cc.settings['master']['calculate_score'] and scores: best_node = sorted( scores, key=lambda x: x[1], reverse=ScoreCalculatorFactory.create().is_reversed)[-1] self._logger.info('Best result: {}:{} = {}'.format( best_node[0]['address'], best_node[0]['port'], best_node[1]))
def optimize_generator_mixture_weights(self): generators = self.neighbourhood.best_generators weights_generators = self.neighbourhood.mixture_weights_generators # Not necessary for single-cell grids, as mixture must always be [1] if self.neighbourhood.grid_size == 1: return # Create random vector from latent space z_noise = noise(self.score_sample_size, generators.individuals[0].genome.data_size) # Include option to start from random weights if self.es_random_init: aux_weights = np.random.rand(len(weights_generators)) aux_weights /= np.sum(aux_weights) weights_generators = OrderedDict( zip(weights_generators.keys(), aux_weights)) self.neighbourhood.mixture_weights_generators = weights_generators dataset = MixedGeneratorDataset( generators, weights_generators, self.score_sample_size, self.mixture_generator_samples_mode, z_noise, ) self.score = self.score_calc.calculate(dataset)[0] init_score = self.score self._logger.info( "Mixture weight mutation - Starting mixture weights optimization ..." ) self._logger.info("Init score: {}\tInit weights: {}.".format( init_score, weights_generators)) for g in range(self.es_generations): # Mutate mixture weights z = np.random.normal(loc=0, scale=self.mixture_sigma, size=len(weights_generators)) transformed = np.asarray( [value for _, value in weights_generators.items()]) transformed += z # Don't allow negative values, normalize to sum of 1.0 transformed = np.clip(transformed, 0, None) transformed /= np.sum(transformed) new_mixture_weights = OrderedDict( zip(weights_generators.keys(), transformed)) # TODO: Testing the idea of not generating the images again dataset = MixedGeneratorDataset( generators, new_mixture_weights, self.score_sample_size, self.mixture_generator_samples_mode, z_noise, epoch=g) if self.score_calc is not None: score_after_mutation = self.score_calc.calculate(dataset)[0] self._logger.info( "Mixture weight mutation - Generation: {} \tScore of new weights: {}\tNew weights: {}." .format(g, score_after_mutation, new_mixture_weights)) # For fid the lower the better, for inception_score, the higher the better if (score_after_mutation < self.score and self.score_calc.is_reversed) or ( score_after_mutation > self.score and (not self.score_calc.is_reversed)): weights_generators = new_mixture_weights self.score = score_after_mutation self._logger.info( "Mixture weight mutation - Generation: {} \tNew score: {}\tWeights changed to: {}." .format(g, self.score, weights_generators)) self.neighbourhood.mixture_weights_generators = weights_generators self._logger.info( "Mixture weight mutation - Score before mixture weight optimzation: {}\tScore after mixture weight optimzation: {}." .format(init_score, self.score))
def create_ensemble(self): n_samples = 50000 using_max_size = self.ensemble_max_size != 0 population = Population(individuals=[], default_fitness=0) sources = [] current_tvd = 1.0 current_fid = 100 current_mixture_definition = dict() generators_examined = 0 self.show_experiment_configuration() start_time = time.time() while True: next_generator_path, source = self.get_next_generator_path() if next_generator_path == '': text = 'Warning: \n' text += 'No more generators to be examined to be added to the ensemble. \n' text += 'Generators examined: {}\n'.format(generators_examined) self.show_file_screen(text) if self.output_file != '': self.show_file_screen(text, self.output_file) break generator = self.network_factory.create_generator() generator.net.load_state_dict(torch.load(next_generator_path, map_location='cpu')) generator.net.eval() population.individuals.append(Individual(genome=generator, fitness=0, source=source)) sources.append(source) ensemble_size = len(population.individuals) tvd_tentative = 1.0 mixture_definition_i = dict() combinations_of_weights, size = self.get_weights_tentative(ensemble_size) if size == 0: break for tentative_mixture_definition in combinations_of_weights: mixture_definition = dict(zip(sources, tentative_mixture_definition)) dataset = MixedGeneratorDataset(population, mixture_definition, n_samples, self.mixture_generator_samples_mode) fid, tvd = self.score_calc.calculate(dataset) if tvd < tvd_tentative: tvd_tentative = tvd fid_tentative = fid mixture_definition_i = mixture_definition generators_examined += 1 text = 'Generators examined={} - Mixture: {} - FID={}, TVD={}, FIDi={}, TVDi={}, FIDbest={}, ' \ 'TVDbest={}'.format(generators_examined, mixture_definition, fid, tvd, fid_tentative, tvd_tentative, current_fid, current_tvd) self.show_file_screen(text) if self.output_file != '': self.show_file_screen(text+ '\n', self.output_file) if tvd_tentative < current_tvd: current_tvd = tvd_tentative current_fid = fid_tentative current_mixture_definition = mixture_definition_i convergence_time = 0 else: sources.pop() population.individuals.pop() convergence_time += 1 if using_max_size and len(sources) == self.ensemble_max_size: break else: if self.max_time_without_improvements!= 0 and convergence_time > self.max_time_without_improvements: break text = 'Finishing execution....\n' text += 'FID={}'.format(current_fid) text += 'TVD={}'.format(current_tvd) text += 'Generators examined={}'.format(generators_examined) text += 'Ensemble: {}'.format(current_mixture_definition) text += 'Execution time={} \n'.format(time.time() - start_time) self.show_file_screen(text) if self.output_file != '': self.show_file_screen(text, self.output_file) # dataset = 'mnist' # precision=10 # mode='random' # ensemble_max_size = 3 # greedy = GreedyEnsembleGenerator(dataset, ensemble_max_size, precision, generators_prefix='mnist-generator', generators_path='./mnist-generators/', # mode=mode) # # greedy.create_ensemble()