def evaluate_block(self, indiv_id, block_index, block_def, block_material,
training_datapair, validation_datapair):
'''
since each block has a slightly different behavior about what exactly get's passed in as data,
I made a generalized evaluate method here that can get called in several different ways in the
other evaluate method
'''
if block_material.need_evaluate:
ezLogging.info(
"%s - Sending to %ith BlockDefinition %s to Evaluate" %
(indiv_id, block_index, block_def.nickname))
if block_def.nickname == 'tensorflow_block':
# don't deepcopy, just work off the same data-instances from transferlearning block
block_def.evaluate(block_material, training_datapair,
validation_datapair)
# delete attributes we don't need that can't be "deepcopy"-ed
del training_datapair.pipeline_wrapper.graph_input_layer
del training_datapair.pipeline_wrapper.final_pretrained_layer
else:
block_def.evaluate(block_material, deepcopy(training_datapair),
deepcopy(validation_datapair))
if block_material.dead:
indiv_material.dead = True
else:
pass
else:
ezLogging.info(
"%s - Didn't need to evaluate %ith BlockDefinition %s" %
(indiv_id, block_index, block_def.nickname))
def mate(self,
parent1: IndividualMaterial,
parent2: IndividualMaterial,
block_def, #: BlockDefinition,
block_index: int):
ezLogging.info("%s+%s-%s - No mating for block %i" % (parent1.id, parent2.id, block_def.nickname, block_index))
return []
def mutate_single_argvalue(mutant_material: BlockMaterial,
block_def): #: BlockDefinition):
'''
instead of looking for a different arg index in .args with the same arg type,
mutate the value stored in this arg index.
'''
ezLogging.info("%s - Inside mutate_single_argvalue" % (mutant_material.id))
if len(mutant_material.active_args) > 0:
# if block has arguments, then there is something to mutate
choices = np.arange(block_def.arg_count)
choices = rnd.choice(choices, size=len(choices),
replace=False) #randomly reorder
for arg_index in choices:
mutant_material.args[arg_index].mutate()
ezLogging.info("%s - Mutated node %i; new arg value: %s" %
(mutant_material.id, arg_index,
mutant_material.args[arg_index]))
if arg_index in mutant_material.active_args:
# active_arg finally mutated
ezLogging.debug("%s - Mutated node %i - active" %
(mutant_material.id, arg_index))
mutant_material.need_evaluate = True
break
else:
ezLogging.debug("%s - Mutated node %i - inactive" %
(mutant_material.id, arg_index))
else:
# won't actually mutate
ezLogging.warning("%s - No active args to mutate" %
(mutant_material.id))
def evaluate(self,
block_material: BlockMaterial,
block_def,#: BlockDefinition,
training_datapair: ezData,
validation_datapair: ezData):
'''
stuff the old code has but unclear why
gpus = tf.config.experimental.list_physical_devices('GPU')
#tf.config.experimental.set_virtual_device_configuration(gpus[0],[
tf.config.experimental.VirtualDeviceConfiguration(memory_limit = 1024*3)
])
'''
ezLogging.info("%s - Start evaluating..." % (block_material.id))
try:
self.build_graph(block_material, block_def, training_datapair)
except Exception as err:
ezLogging.critical("%s - Build Graph; Failed: %s" % (block_material.id, err))
block_material.dead = True
import pdb; pdb.set_trace()
return
try:
output = self.train_graph(block_material, block_def, training_datapair, validation_datapair)
except Exception as err:
ezLogging.critical("%s - Train Graph; Failed: %s" % (block_material.id, err))
block_material.dead = True
import pdb; pdb.set_trace()
return
block_material.output = output # TODO make sure it is a list
def evaluate(
self,
indiv_material: IndividualMaterial,
indiv_def, #: IndividualDefinition,
training_datapair: ezData,
validation_datapair: ezData = None):
for block_index, (block_material, block_def) in enumerate(
zip(indiv_material.blocks, indiv_def.block_defs)):
if block_material.need_evaluate:
ezLogging.info(
"%s - Sending to %ith BlockDefinition %s to Evaluate" %
(indiv_material.id, block_index, block_def.nickname))
block_def.evaluate(block_material, deepcopy(training_datapair),
deepcopy(validation_datapair))
if block_material.dead:
indiv_material.dead = True
break
else:
pass
else:
ezLogging.info(
"%s - Didn't need to evaluate %ith BlockDefinition %s" %
(indiv_material.id, block_index, block_def.nickname))
training_datapair = block_material.output
indiv_material.output = block_material.output
def evaluate(
self,
indiv_material: IndividualMaterial,
indiv_def, #IndividualDefinition,
training_datapair: ezData,
validation_datapair: ezData):
'''
we want to deepcopy the data before evaluate so that in future if need_evaluate is False, we can grab the
block_material.output and it will be unique to that block not shared with whole individual.
'''
for block_index, (block_material, block_def) in enumerate(
zip(indiv_material.blocks, indiv_def.block_defs)):
if block_material.need_evaluate:
ezLogging.info(
"%s - Sending to %ith BlockDefinition %s to Evaluate" %
(indiv_material.id, block_index, block_def.nickname))
block_def.evaluate(block_material, deepcopy(training_datapair),
deepcopy(validation_datapair))
if block_material.dead:
indiv_material.dead = True
break
else:
pass
else:
ezLogging.info(
"%s - Didn't need to evaluate %ith BlockDefinition %s" %
(indiv_material.id, block_index, block_def.nickname))
training_datapair, validation_datapair = block_material.output
indiv_material.output = training_datapair, validation_datapair
def get_actives(self, block_material: BlockMaterial):
'''
method will go through and set the attributes block_material.active_nodes and active_args.
active_nodes will include all output_nodes, a subset of main_nodes and input_nodes.
'''
ezLogging.info("%s - Inside get_actives" % (block_material.id))
block_material.active_nodes = set(np.arange(self.main_count, self.main_count+self.output_count))
block_material.active_args = set()
#block_material.active_ftns = set()
# add feeds into the output_nodes
for node_input in range(self.main_count, self.main_count+self.output_count):
block_material.active_nodes.update([block_material[node_input]])
for node_index in reversed(range(self.main_count)):
if node_index in block_material.active_nodes:
# then add the input nodes to active list
block_material.active_nodes.update(block_material[node_index]["inputs"])
block_material.active_args.update(block_material[node_index]["args"])
else:
pass
# sort
block_material.active_nodes = sorted(list(block_material.active_nodes))
ezLogging.debug("%s - active nodes: %s" % (block_material.id, block_material.active_nodes))
block_material.active_args = sorted(list(block_material.active_args))
ezLogging.debug("%s - active args: %s" % (block_material.id, block_material.active_args))
def mate(self, parent1: IndividualMaterial, parent2: IndividualMaterial, block_index: int):
'''
wrapper method to call the block's mate definition
'''
ezLogging.info("%s+%s-%s - Sending to Block Mate Definition" % (parent1.id, parent2.id, self.nickname))
children = self.mate_def.mate(parent1, parent2, self, block_index)
ezLogging.debug("%s+%s-%s - Received %i Children from Block Mate Definition" % (parent1.id, parent2.id, self.nickname, len(children)))
return children
def postprocess_generation(self, universe):
'''
after each generation, we want to save the scores (plot performance over time)
and save the population for seeding
'''
ezLogging.info("Post Processing Generation Run - saving")
save_things.save_fitness_scores(universe)
save_things.save_population(universe)
def mate(self,
parent1: IndividualMaterial,
parent2: IndividualMaterial,
block_def,#: BlockDefinition,
block_index: int):
ezLogging.info("%s+%s-%s - Sending %i block to mate_methods.whole_block()" % (parent1.id, parent2.id, block_def.nickname, block_index))
# dont actually need block_def
return mate_methods.whole_block(parent1, parent2, block_index)
def mutate(self, indiv_material: IndividualMaterial):
'''
wrapper method that just directs mutate call to the IndividualMutate class definition of mutate
'''
ezLogging.info("%s - Sending to Individual Mutate Definition" %
(indiv_material.id))
mutants = self.mutate_def.mutate(indiv_material, self)
return mutants
def mate(self, parent1: IndividualMaterial, parent2: IndividualMaterial):
'''
wrapper method that just directs mate call to the IndividualMate class definition of mate
'''
ezLogging.info("%s+%s - Sending to Individuals Mate Definition" %
(parent1.id, parent2.id))
children = self.mate_def.mate(parent1, parent2, self)
return children
def merge_subpopulations(self, subpops):
'''
if we had a list of list of individual_materials in subpops,
then we'd want to append them into a single large list and
assign to self.population
'''
self.population = list(itertools.chain.from_iterable(subpops))
ezLogging.info("Combined %i sub populations into a single population" %
(len(subpops)))
def mutate(self, mutant_material: BlockMaterial,
block_def): #: BlockDefinition):
roll = rnd.random()
ezLogging.info("%s - Sending block to mutate; roll: %f" %
(mutant_material.id, roll))
if roll < (1 / 2):
mutate_methods.mutate_single_input(mutant_material, block_def)
else:
mutate_methods.mutate_single_ftn(mutant_material, block_def)
def postprocess_universe(self, universe):
'''
NOTE that this is not an abstractmethod because the user may choose not to do anything here
the idea here is that the universe.run() is about to exit but before it does,
we can export or plot things wrt the final population
'''
ezLogging.info("Post Processing Universe Run - pass")
pass
def evaluate(self,
block_material: BlockMaterial,
block_def,#: BlockDefinition,
training_datapair: ezData,
validation_datapair: ezData=None):
ezLogging.info("%s - Start evaluating..." % (block_material.id))
output_list = self.standard_evaluate(block_material, block_def, training_datapair.x)
#training_datapair.x = output_list[0]
#block_material.output = training_datapair
block_material.output = output_list
def mutate_population(self, problem: ProblemDefinition_Abstract):
'''
super simple...just loop through am call mutate. at the block level is where it decides to mutate or not
'''
start_time = time.time()
mutants = []
for individual in self.population.population:
mutants += problem.indiv_def.mutate(individual)
self.population.add_next_generation(mutants)
ezLogging.info("Node %i - Mutation took %.2f minutes" %
(self.node_number, (time.time() - start_time) / 60))
def evaluate(self,
indiv_material: IndividualMaterial,
training_datapair: ezData,
validation_datapair=None):
'''
wrapper method that just directs evaluate call to the IndividualEvaluate class definition of mate
'''
ezLogging.info("%s - Sending to Individual Evaluate Definition" %
(indiv_material.id))
self.evaluate_def.evaluate(indiv_material, self, training_datapair,
validation_datapair)
def evaluate(self,
block_material: BlockMaterial,
block_def, #: BlockDefinition,
training_datapair: ezData,
validation_datapair: ezData):
ezLogging.info("%s - Start evaluating..." % (block_material.id))
output_list = self.standard_evaluate(block_material, block_def, [training_datapair.pipeline])
training_datapair.pipeline = output_list[0] #assuming only outputs the pipeline
block_material.output = [training_datapair, validation_datapair]
def standard_build_graph(self,
block_material: BlockMaterial,
block_def,#: BlockDefinition,
input_layers = None):
'''
trying to generalize the graph building process similar to standard_evaluate()
For Transfer Learning:
we expect input_layers to be None, and later when we call function(*inputs, *args), we want to pass in
an empty list for inputs.
This also guarentees that no matter how many 'active nodes' we have for our transfer learning block,
we will only ever use one pretrained model...no inputs are shared between nodes so the models never connect!
'''
# add input data
if input_layers is not None:
for i, input_layer in enumerate(input_layers):
block_material.evaluated[-1*(i+1)] = input_layer
# go solve
for node_index in block_material.active_nodes:
if node_index < 0:
# do nothing. at input node
continue
elif node_index >= block_def.main_count:
# do nothing NOW. at output node. we'll come back to grab output after this loop
continue
else:
# main node. this is where we evaluate
function = block_material[node_index]["ftn"]
inputs = []
if input_layers is not None:
node_input_indices = block_material[node_index]["inputs"]
for node_input_index in node_input_indices:
inputs.append(block_material.evaluated[node_input_index])
ezLogging.debug("%s - Eval %i; input index: %s" % (block_material.id, node_index, node_input_indices))
args = []
node_arg_indices = block_material[node_index]["args"]
for node_arg_index in node_arg_indices:
args.append(block_material.args[node_arg_index].value)
ezLogging.debug("%s - Eval %i; arg index: %s, value: %s" % (block_material.id, node_index, node_arg_indices, args))
ezLogging.debug("%s - Eval %i; Function: %s, Inputs: %s, Args: %s" % (block_material.id, node_index, function, inputs, args))
block_material.evaluated[node_index] = function(*inputs, *args)
output = []
if not block_material.dead:
for output_index in range(block_def.main_count, block_def.main_count+block_def.output_count):
output.append(block_material.evaluated[block_material.genome[output_index]])
ezLogging.info("%s - Ending evaluating...%i output" % (block_material.id, len(output)))
return output
def evolve_population(self, problem: ProblemDefinition_Abstract):
'''
TODO
'''
# MATE
self.mate_population(problem)
ezLogging.info("Population size after Mating: %i" %
(len(self.population.population)))
# MUTATE
self.mutate_population(problem)
ezLogging.info("Population size after Mutating: %i" %
(len(self.population.population)))
def mate_population(self, problem: ProblemDefinition_Abstract):
'''
do a ranking/sorting of parents, then pair them off, mate the pairs, return and add the children
'''
start_time = time.time()
children = []
mating_list = self.parent_selection()
for ith_indiv in range(0, len(mating_list), 2):
parent1 = mating_list[ith_indiv]
parent2 = mating_list[ith_indiv + 1]
children += problem.indiv_def.mate(parent1, parent2)
self.population.add_next_generation(children)
ezLogging.info("Node %i - Mating took %.2f minutes" %
(self.node_number, (time.time() - start_time) / 60))
def whole_block(parent1: IndividualMaterial, parent2: IndividualMaterial,
block_index: int):
'''
Super simple direct swaping of the blocks. 2 parents in; 2 children out.
'''
ezLogging.info("%s+%s - Mating Block %i with whole_block()" %
(parent1.id, parent2.id, block_index))
child1 = deepcopy(parent1)
child1[block_index] = deepcopy(parent2[block_index])
child2 = deepcopy(parent2)
child2[block_index] = deepcopy(parent1[block_index])
return [child1, child2]
def partial_block(
parent1: IndividualMaterial,
parent2: IndividualMaterial,
block_def, #: BlockDefinition,
block_index: int):
'''
TODO
'''
ezLogging.info("%s+%s - Mating Block %i with partial_block()" %
(parent1.id, parent2.id, block_index))
child1 = deepcopy(parent1)
child2 = deepcopy(parent2)
# TODO
#return [child1, child2]
return []
def mutate_single_argindex(mutant_material: BlockMaterial,
block_def): #: BlockDefinition):
'''
search through the args and try to find a matching arg_type and use that arg index instead
'''
ezLogging.info("%s - Inside mutate_single_argindex" % (mutant_material.id))
if len(mutant_material.active_args) > 0:
# then there is something to mutate
choices = [] # need to find those nodes with 'args' filled
#weights = [] # option to sample node_index by the number of args for each node
for node_index in range(block_def.main_count):
if len(mutant_material[node_index]["args"]) > 0:
choices.append(node_index)
#weights.append(len(mutant_material[node_index]["args"]))
else:
pass
choices = rnd.choice(choices, size=len(choices),
replace=False) #randomly reorder
for node_index in choices:
ith_arg = rnd.choice(
np.arange(len(mutant_material[node_index]["args"])))
current_arg = mutant_material[node_index]["args"][ith_arg]
arg_dtype = block_def.get_node_dtype(mutant_material, node_index,
"args")[ith_arg]
new_arg = block_def.get_random_arg(arg_dtype,
exclude=[current_arg])
if new_arg is None:
# failed to find a new_arg
continue
else:
mutant_material[node_index]["args"][ith_arg] = new_arg
ezLogging.info(
"%s - Mutated node %i; ori arg index: %i, new arg index: %i"
% (mutant_material.id, node_index, current_arg, new_arg))
if node_index in mutant_material.active_nodes:
# active_node finally mutated
ezLogging.debug("%s - Mutated node %i - active" %
(mutant_material.id, node_index))
mutant_material.need_evaluate = True
break
else:
ezLogging.debug("%s - Mutated node %i - inactive" %
(mutant_material.id, node_index))
else:
# won't actually mutate
ezLogging.warning("%s - No active args to mutate" %
(mutant_material.id))
def evaluate(self,
block_material: BlockMaterial,
block_def,#: BlockDefinition,
training_datapair: ezData,
validation_datapair: ezData):
ezLogging.info("%s - Start evaluating..." % (block_material.id))
try:
self.build_graph(block_material, block_def, training_datapair)
except Exception as err:
ezLogging.critical("%s - Build Graph; Failed: %s" % (block_material.id, err))
block_material.dead = True
import pdb; pdb.set_trace()
return
block_material.output = [training_datapair, validation_datapair]
def run(self, problem: ProblemDefinition_Abstract):
'''
assumes a population has only been created and not evaluatedscored
'''
self.generation = 0
self.population = self.factory.build_population(
problem.indiv_def, problem.pop_size, problem.genome_seeds)
self.evaluate_score_population(problem)
self.population_selection(problem)
while not self.converged:
self.generation += 1
ezLogging.info("Starting Generation %i" % self.generation)
self.evolve_population(problem)
self.evaluate_score_population(problem)
self.population_selection(problem)
self.check_convergence(problem)
self.postprocess_generation(problem)
self.postprocess_universe(problem)
def evaluate(self,
block_material: BlockMaterial,
block_def,#: BlockDefinition,
training_datapair: ezData,
validation_datapair: ezData):
ezLogging.info("%s - Start evaluating..." % (block_material.id))
# going to treat training + validation as separate block_materials!
output = []
for datapair in [training_datapair, validation_datapair]:
single_output_list = self.standard_evaluate(block_material, block_def, [datapair.pipeline])
datapair.pipeline = single_output_list[0]
if block_material.dead:
return []
else:
output.append(datapair)
self.preprocess_block_evaluate(block_material) #prep for next loop through datapair
block_material.output = output
def mpi_mate_population(self, problem: ProblemDefinition_Abstract):
'''
the only difference here with mate_population() [<-no mpi] is that we do parent_selection before hand
because by this point we already split and scattered the population to each node and we'd prefer to do
parent selection amoung the whole population
earlier at __init__ we should have adjusted pop size to be a multiple of 4 and multiples of our node count
so this should gaurentee that parents do not get split up when we split+scatter to the different nodes from root=0
'''
start_time = time.time()
children = []
mating_list = self.population.population
ezLogging.debug("here mating %i" % (len(mating_list)))
for ith_indiv in range(0, len(mating_list), 2):
parent1 = mating_list[ith_indiv]
parent2 = mating_list[ith_indiv + 1]
children += problem.indiv_def.mate(parent1, parent2)
self.population.add_next_generation(children)
ezLogging.info("Node %i - Mating took %.2f minutes" %
(self.node_number, (time.time() - start_time) / 60))
def evaluate_score_population(self,
problem: ProblemDefinition_Abstract,
compute_node: int = None):
'''
TODO
'''
self.pop_fitness_scores = []
self.pop_individual_ids = []
ezLogging.info("Evaluating Population of size %i" %
(len(self.population.population)))
for indiv in self.population.population:
# EVALUATE
problem.indiv_def.evaluate(indiv, problem.train_data,
problem.validate_data)
# SCORE
problem.objective_functions(indiv)
self.pop_fitness_scores.append(indiv.fitness.values)
# ATTACH ID
self.pop_individual_ids.append(indiv.id)
self.pop_fitness_scores = np.array(self.pop_fitness_scores)
self.pop_individual_ids = np.array(self.pop_individual_ids)
