def mutate_mixture_weights_with_score(self, input_data):
if self.score_calc is not None:
self._logger.info('Calculating FID/inception score.')
# Not necessary for single-cell grids, as mixture must always be [1]
if self.neighbourhood.grid_size == 1:
best_generators = self.neighbourhood.best_generators
dataset = MixedGeneratorDataset(
best_generators,
self.neighbourhood.mixture_weights_generators,
self.score_sample_size, self.cc.settings['trainer']
['mixture_generator_samples_mode'])
self.score = self.score_calc.calculate(dataset)[0]
else:
# Mutate mixture weights
z = np.random.normal(
loc=0,
scale=self.mixture_sigma,
size=len(self.neighbourhood.mixture_weights_generators))
transformed = np.asarray([
value for _, value in
self.neighbourhood.mixture_weights_generators.items()
])
transformed += z
# Don't allow negative values, normalize to sum of 1.0
transformed = np.clip(transformed, 0, None)
transformed /= np.sum(transformed)
new_mixture_weights_generators = OrderedDict(
zip(self.neighbourhood.mixture_weights_generators.keys(),
transformed))
best_generators = self.neighbourhood.best_generators
dataset_before_mutation = MixedGeneratorDataset(
best_generators,
self.neighbourhood.mixture_weights_generators,
self.score_sample_size, self.cc.settings['trainer']
['mixture_generator_samples_mode'])
score_before_mutation = self.score_calc.calculate(
dataset_before_mutation)[0]
del dataset_before_mutation
dataset_after_mutation = MixedGeneratorDataset(
best_generators, new_mixture_weights_generators,
self.score_sample_size, self.cc.settings['trainer']
['mixture_generator_samples_mode'])
score_after_mutation = self.score_calc.calculate(
dataset_after_mutation)[0]
del dataset_after_mutation
# For fid the lower the better, for inception_score, the higher the better
if (score_after_mutation < score_before_mutation and self.score_calc.is_reversed) \
or (score_after_mutation > score_before_mutation and (not self.score_calc.is_reversed)):
# Adopt the mutated mixture_weights only if the performance after mutation is better
self.neighbourhood.mixture_weights_generators = new_mixture_weights_generators
self.score = score_after_mutation
else:
# Do not adopt the mutated mixture_weights here
self.score = score_before_mutation
def generate_samples(args, cc):
print("generating samples")
batch_size = 100
mixture_source = args.mixture_source
output_dir = args.output_dir
sample_size = args.sample_size
dataloader = cc.create_instance(cc.settings['dataloader']['dataset_name'])
network_factory = cc.create_instance(cc.settings['network']['name'],
dataloader.n_input_neurons)
population = Population(individuals=[], default_fitness=0)
mixture_definition = read_settings(
os.path.join(mixture_source, 'mixture.yml'))
for source, weight in mixture_definition.items():
path = os.path.join(mixture_source, source)
generator = network_factory.create_generator()
generator.net.load_state_dict(torch.load(path))
generator.net.eval()
population.individuals.append(
Individual(genome=generator, fitness=0, source=source))
dataset = MixedGeneratorDataset(
population, mixture_definition, sample_size * batch_size,
cc.settings['trainer']['mixture_generator_samples_mode'])
os.makedirs(output_dir, exist_ok=True)
LipizzanerMaster().save_samples(dataset, output_dir, dataloader,
sample_size, batch_size)
def calc_score(args, cc):
score_calc = ScoreCalculatorFactory.create()
cc.settings['general']['distribution']['client_id'] = 0
dataloader = cc.create_instance(cc.settings['dataloader']['dataset_name'])
network_factory = cc.create_instance(cc.settings['network']['name'],
dataloader.n_input_neurons)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
generator = network_factory.create_generator()
generator.net.load_state_dict(
torch.load(args.generator_file, map_location=device))
generator.net.eval()
individual = Individual(genome=generator, fitness=0, source='local')
dataset = MixedGeneratorDataset(
Population(individuals=[individual],
default_fitness=0), {'local': 1.0}, 50000,
cc.settings['trainer']['mixture_generator_samples_mode'])
output_dir = os.path.join(cc.output_dir, 'score')
os.makedirs(output_dir, exist_ok=True)
LipizzanerMaster().save_samples(dataset, output_dir, dataloader)
inc = score_calc.calculate(dataset)
_logger.info('Generator loaded from \'{}\' yielded a score of {}'.format(
args.generator_file, inc))
def calculate_score(self):
best_generators = self.neighbourhood.best_generators
dataset = MixedGeneratorDataset(
best_generators, self.neighbourhood.mixture_weights_generators,
self.score_sample_size)
self.score = self.score_calc.calculate(dataset)[0]
def compute_mixture_generative_score(self, epoch=None):
# Not necessary for single-cell grids, as mixture must always be [1]
self._logger.info("Calculating score {}.".format(epoch))
best_generators = self.neighbourhood.best_generators
if True or self.neighbourhood.grid_size == 1:
if self.score_calc is not None:
dataset = MixedGeneratorDataset(
best_generators,
self.neighbourhood.mixture_weights_generators,
self.score_sample_size,
self.cc.settings["trainer"]
["mixture_generator_samples_mode"],
# epoch=epoch
)
self.score = self.score_calc.calculate(dataset)[0]
def calc_inception_score(args, cc):
inception_cal = InceptionCalculator(cuda=True)
dataloader = cc.create_instance(cc.settings['dataloader']['dataset_name'])
network_factory = cc.create_instance(cc.settings['network']['name'],
dataloader.n_input_neurons)
generator = network_factory.create_generator()
generator.net.load_state_dict(torch.load(args.inception_file))
generator.net.eval()
individual = Individual(genome=generator, fitness=0, source='local')
dataset = MixedGeneratorDataset(
Population(individuals=[individual], default_fitness=0),
{'local': 1.0}, 50000)
output_dir = os.path.join(cc.output_dir, 'inception_score')
os.makedirs(output_dir, exist_ok=True)
LipizzanerMaster().save_samples(dataset, output_dir, dataloader)
inc = inception_cal.calculate(dataset)
_logger.info(
'Generator loaded from \'{}\' yielded an inception score of {}'.format(
args.inception_file, inc))
def evaluate_ensemble(self,
individual,
network_factory,
mixture_generator_samples_mode='exact_proportion',
fitness_type='tvd'):
"""It evaluates the solution/individual (ensemble) given the fitness type. It generates samples and it evaluates
the metric defined by fitness_type using Lipizzaner.
:parameter individual: Solutionto be evaluated
:parameter network_factory:
:parameter mixture_generator_samples_mode:
:parameter fitness_type: It defines the type of metric to be evaluated.
:return: The fitness_type metric value got by the solution.
"""
population = Population(individuals=[], default_fitness=0)
# weight_and_generator_indices = [math.modf(gen) for gen in individual]
# generators_paths, sources = self.ga.get_generators_for_ensemble(weight_and_generator_indices)
# tentative_weights = [weight for weight, generator_index in weight_and_generator_indices]
tentative_weights, generators_paths, sources = self.ga.get_mixture_from_individual(
individual)
mixture_definition = dict(zip(sources, tentative_weights))
for path, source in zip(generators_paths, sources):
generator = network_factory.create_generator()
generator.net.load_state_dict(torch.load(path, map_location='cpu'))
generator.net.eval()
population.individuals.append(
Individual(genome=generator, fitness=0, source=source))
dataset = MixedGeneratorDataset(population, mixture_definition, 50000,
mixture_generator_samples_mode)
fid, tvd = self.score_calc.calculate(dataset)
if fitness_type == 'tvd':
return tvd,
elif fitness_type == 'fid':
return fid,
elif fitness_type == 'tvd-fid':
return (tvd, fid),
def _gather_results(self):
self._logger.info('Collecting results from clients...')
# Initialize node client
dataloader = self.cc.create_instance(
self.cc.settings['dataloader']['dataset_name'])
network_factory = self.cc.create_instance(
self.cc.settings['network']['name'], dataloader.n_input_neurons)
node_client = NodeClient(network_factory)
db_logger = DbLogger()
results = node_client.gather_results(
self.cc.settings['general']['distribution']['client_nodes'], 120)
scores = []
for (node, generator_pop, discriminator_pop, weights_generator,
weights_discriminator) in results:
node_name = '{}:{}'.format(node['address'], node['port'])
try:
output_dir = self.get_and_create_output_dir(node)
for generator in generator_pop.individuals:
source = generator.source.replace(':', '-')
filename = '{}{}.pkl'.format(GENERATOR_PREFIX, source)
torch.save(
generator.genome.net.state_dict(),
os.path.join(output_dir,
'generator-{}.pkl'.format(source)))
with open(os.path.join(output_dir, 'mixture.yml'),
"a") as file:
file.write('{}: {}\n'.format(
filename, weights_generator[generator.source]))
for discriminator in discriminator_pop.individuals:
source = discriminator.source.replace(':', '-')
filename = '{}{}.pkl'.format(DISCRIMINATOR_PREFIX, source)
torch.save(discriminator.genome.net.state_dict(),
os.path.join(output_dir, filename))
# Save images
dataset = MixedGeneratorDataset(
generator_pop, weights_generator,
self.cc.settings['master']['score_sample_size'],
self.cc.settings['trainer']
['mixture_generator_samples_mode'])
image_paths = self.save_samples(dataset, output_dir,
dataloader)
self._logger.info(
'Saved mixture result images of client {} to target directory {}.'
.format(node_name, output_dir))
# Calculate inception or FID score
score = float('-inf')
if self.cc.settings['master']['calculate_score']:
calc = ScoreCalculatorFactory.create()
self._logger.info('Score calculator: {}'.format(
type(calc).__name__))
self._logger.info(
'Calculating score score of {}. Depending on the type, this may take very long.'
.format(node_name))
score = calc.calculate(dataset)
self._logger.info(
'Node {} with weights {} yielded a score of {}'.format(
node_name, weights_generator, score))
scores.append((node, score))
if db_logger.is_enabled and self.experiment_id is not None:
db_logger.add_experiment_results(self.experiment_id,
node_name, image_paths,
score)
except Exception as ex:
self._logger.error(
'An error occured while trying to gather results from {}: {}'
.format(node_name, ex))
traceback.print_exc()
if self.cc.settings['master']['calculate_score'] and scores:
best_node = sorted(
scores,
key=lambda x: x[1],
reverse=ScoreCalculatorFactory.create().is_reversed)[-1]
self._logger.info('Best result: {}:{} = {}'.format(
best_node[0]['address'], best_node[0]['port'], best_node[1]))
def optimize_generator_mixture_weights(self):
generators = self.neighbourhood.best_generators
weights_generators = self.neighbourhood.mixture_weights_generators
# Not necessary for single-cell grids, as mixture must always be [1]
if self.neighbourhood.grid_size == 1:
return
# Create random vector from latent space
z_noise = noise(self.score_sample_size,
generators.individuals[0].genome.data_size)
# Include option to start from random weights
if self.es_random_init:
aux_weights = np.random.rand(len(weights_generators))
aux_weights /= np.sum(aux_weights)
weights_generators = OrderedDict(
zip(weights_generators.keys(), aux_weights))
self.neighbourhood.mixture_weights_generators = weights_generators
dataset = MixedGeneratorDataset(
generators,
weights_generators,
self.score_sample_size,
self.mixture_generator_samples_mode,
z_noise,
)
self.score = self.score_calc.calculate(dataset)[0]
init_score = self.score
self._logger.info(
"Mixture weight mutation - Starting mixture weights optimization ..."
)
self._logger.info("Init score: {}\tInit weights: {}.".format(
init_score, weights_generators))
for g in range(self.es_generations):
# Mutate mixture weights
z = np.random.normal(loc=0,
scale=self.mixture_sigma,
size=len(weights_generators))
transformed = np.asarray(
[value for _, value in weights_generators.items()])
transformed += z
# Don't allow negative values, normalize to sum of 1.0
transformed = np.clip(transformed, 0, None)
transformed /= np.sum(transformed)
new_mixture_weights = OrderedDict(
zip(weights_generators.keys(), transformed))
# TODO: Testing the idea of not generating the images again
dataset = MixedGeneratorDataset(
generators,
new_mixture_weights,
self.score_sample_size,
self.mixture_generator_samples_mode,
z_noise,
epoch=g)
if self.score_calc is not None:
score_after_mutation = self.score_calc.calculate(dataset)[0]
self._logger.info(
"Mixture weight mutation - Generation: {} \tScore of new weights: {}\tNew weights: {}."
.format(g, score_after_mutation, new_mixture_weights))
# For fid the lower the better, for inception_score, the higher the better
if (score_after_mutation < self.score
and self.score_calc.is_reversed) or (
score_after_mutation > self.score and
(not self.score_calc.is_reversed)):
weights_generators = new_mixture_weights
self.score = score_after_mutation
self._logger.info(
"Mixture weight mutation - Generation: {} \tNew score: {}\tWeights changed to: {}."
.format(g, self.score, weights_generators))
self.neighbourhood.mixture_weights_generators = weights_generators
self._logger.info(
"Mixture weight mutation - Score before mixture weight optimzation: {}\tScore after mixture weight optimzation: {}."
.format(init_score, self.score))
def create_ensemble(self):
n_samples = 50000
using_max_size = self.ensemble_max_size != 0
population = Population(individuals=[], default_fitness=0)
sources = []
current_tvd = 1.0
current_fid = 100
current_mixture_definition = dict()
generators_examined = 0
self.show_experiment_configuration()
start_time = time.time()
while True:
next_generator_path, source = self.get_next_generator_path()
if next_generator_path == '':
text = 'Warning: \n'
text += 'No more generators to be examined to be added to the ensemble. \n'
text += 'Generators examined: {}\n'.format(generators_examined)
self.show_file_screen(text)
if self.output_file != '': self.show_file_screen(text, self.output_file)
break
generator = self.network_factory.create_generator()
generator.net.load_state_dict(torch.load(next_generator_path, map_location='cpu'))
generator.net.eval()
population.individuals.append(Individual(genome=generator, fitness=0, source=source))
sources.append(source)
ensemble_size = len(population.individuals)
tvd_tentative = 1.0
mixture_definition_i = dict()
combinations_of_weights, size = self.get_weights_tentative(ensemble_size)
if size == 0:
break
for tentative_mixture_definition in combinations_of_weights:
mixture_definition = dict(zip(sources, tentative_mixture_definition))
dataset = MixedGeneratorDataset(population,
mixture_definition,
n_samples,
self.mixture_generator_samples_mode)
fid, tvd = self.score_calc.calculate(dataset)
if tvd < tvd_tentative:
tvd_tentative = tvd
fid_tentative = fid
mixture_definition_i = mixture_definition
generators_examined += 1
text = 'Generators examined={} - Mixture: {} - FID={}, TVD={}, FIDi={}, TVDi={}, FIDbest={}, ' \
'TVDbest={}'.format(generators_examined, mixture_definition, fid, tvd, fid_tentative,
tvd_tentative, current_fid, current_tvd)
self.show_file_screen(text)
if self.output_file != '': self.show_file_screen(text+ '\n', self.output_file)
if tvd_tentative < current_tvd:
current_tvd = tvd_tentative
current_fid = fid_tentative
current_mixture_definition = mixture_definition_i
convergence_time = 0
else:
sources.pop()
population.individuals.pop()
convergence_time += 1
if using_max_size and len(sources) == self.ensemble_max_size:
break
else:
if self.max_time_without_improvements!= 0 and convergence_time > self.max_time_without_improvements:
break
text = 'Finishing execution....\n'
text += 'FID={}'.format(current_fid)
text += 'TVD={}'.format(current_tvd)
text += 'Generators examined={}'.format(generators_examined)
text += 'Ensemble: {}'.format(current_mixture_definition)
text += 'Execution time={} \n'.format(time.time() - start_time)
self.show_file_screen(text)
if self.output_file != '': self.show_file_screen(text, self.output_file)
# dataset = 'mnist'
# precision=10
# mode='random'
# ensemble_max_size = 3
# greedy = GreedyEnsembleGenerator(dataset, ensemble_max_size, precision, generators_prefix='mnist-generator', generators_path='./mnist-generators/',
# mode=mode)
#
# greedy.create_ensemble()
