def build_graph(self, block_material, block_def, datapair):
'''
Assume input+output layers are going to be lists with only one element
https://www.tensorflow.org/api_docs/python/tf/keras/layers/InputLayer
vs
https://www.tensorflow.org/api_docs/python/tf/keras/Input
'''
ezLogging.debug("%s - Building Graph" % (block_material.id))
output_layer = self.standard_build_graph(block_material,
block_def,
[datapair.final_pretrained_layer])[0]
# flatten the output node and perform a softmax
output_flatten = tf.keras.layers.Flatten()(output_layer)
logits = tf.keras.layers.Dense(units=datapair.num_classes, activation=None, use_bias=True)(output_flatten)
softmax = tf.keras.layers.Softmax(axis=1)(logits) # TODO verify axis...axis=1 was given by original code
#https://www.tensorflow.org/api_docs/python/tf/keras/Model
block_material.graph = tf.keras.Model(inputs=datapair.graph_input_layer, outputs=softmax)
#https://www.tensorflow.org/api_docs/python/tf/keras/Model#compile
block_material.graph.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),
loss="categorical_crossentropy",
metrics=[tf.keras.metrics.Accuracy(),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall()],
loss_weights=None,
weighted_metrics=None,
run_eagerly=None)
def mutate_single_argvalue(mutant_material: BlockMaterial,
block_def): #: BlockDefinition):
'''
instead of looking for a different arg index in .args with the same arg type,
mutate the value stored in this arg index.
'''
ezLogging.info("%s - Inside mutate_single_argvalue" % (mutant_material.id))
if len(mutant_material.active_args) > 0:
# if block has arguments, then there is something to mutate
choices = np.arange(block_def.arg_count)
choices = rnd.choice(choices, size=len(choices),
replace=False) #randomly reorder
for arg_index in choices:
mutant_material.args[arg_index].mutate()
ezLogging.info("%s - Mutated node %i; new arg value: %s" %
(mutant_material.id, arg_index,
mutant_material.args[arg_index]))
if arg_index in mutant_material.active_args:
# active_arg finally mutated
ezLogging.debug("%s - Mutated node %i - active" %
(mutant_material.id, arg_index))
mutant_material.need_evaluate = True
break
else:
ezLogging.debug("%s - Mutated node %i - inactive" %
(mutant_material.id, arg_index))
else:
# won't actually mutate
ezLogging.warning("%s - No active args to mutate" %
(mutant_material.id))
def train_graph(self,
block_material,
block_def,
training_datapair,
validation_datapair):
ezLogging.debug("%s - Building Generators" % (block_material.id))
training_generator, validation_generator = self.get_generator(block_material,
block_def,
training_datapair,
validation_datapair)
ezLogging.debug("%s - Training Graph - %i batch size, %i steps, %i epochs" % (block_material.id,
block_def.batch_size,
training_datapair.num_images//block_def.batch_size,
block_def.epochs))
history = block_material.graph.fit(x=training_generator,
epochs=block_def.epochs,
verbose=2, # TODO set to 0 after done debugging
callbacks=None,
validation_data=validation_generator,
shuffle=True,
steps_per_epoch=training_datapair.num_images//block_def.batch_size, # TODO
validation_steps=validation_datapair.num_images//block_def.batch_size,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
)
tf.keras.backend.clear_session()
output = history.stuff # validation metrics
return [-1 * history.history['val_accuracy'][-1], #mult by -1 since we want to maximize accuracy but universe optimization is minimization of fitness
-1 * history.history['val_precision'][-1],
-1 * history.history['val_recall'][-1]]
def set_arg_types(self):
'''
given a list of unique argument data types and another list giving the percent share of the
arg_count, this method will fill out a list of .arg_types to be used to initialize the .arg
of blocks/individuals.
'''
ezLogging.debug("%s-%s - Inside set_arg_types" % (None, None))
start_point = 0
end_point = 0
self.arg_types = [None] * self.arg_count
for arg_class, arg_weight in zip(self.each_type, self.each_weight):
end_point += int(arg_weight * self.arg_count)
for arg_index in range(start_point, end_point):
self.arg_types[arg_index] = arg_class
start_point = end_point
if end_point != self.arg_count:
# prob some rounding errors then
sorted_byweight = np.argsort(
self.each_weight
)[::-1] # sort then reverse to go from largest to smallest
for i, arg_index in enumerate(range(end_point, self.arg_count)):
arg_class = self.each_type[sorted_byweight[i]]
self.arg_types[arg_index] = arg_class
else:
pass
def set_equal_weights(self, module):
ezLogging.debug("%s-%s - Inside set_equal_weights" % (None, None))
weight_dict = {}
for func in self.get_all_functions(module):
weight_dict[func] = 1
return weight_dict
def __init__(self, value=None):
if value is None:
self.value = None
self.mutate()
else:
self.value = value
ezLogging.debug("%s-%s - Initialize ArgumentType_LimitedFloat0to1 Class to %f" % (None, None, self.value))
def mutate(self):
roll = rnd.random()
if roll < 2/3:
self.mutate_unif_int10()
else:
self.mutate_unif_localint()
ezLogging.debug("%s-%s - Mutated ArgumentType_Int1to10 to %f" % (None, None, self.value))
def mutate(self):
delta = 0.05
choices = list(np.arange(0, 1, delta) + delta) #[0.05, 0.1, ..., 0.95, 1.0]
if self.value in choices:
choices.remove(self.value) # works in-place
self.value = np.random.choice(choices)
ezLogging.debug("%s-%s - Mutated ArgumentType_LimitedFloat0to1 to %f" % (None, None, self.value))
def get_actives(self, indiv_material: IndividualMaterial):
'''
loop over each block and set the actives attribute to prep for evaluation
'''
ezLogging.debug("%s - Inside get_actives" % (indiv_material.id))
for block_index in range(self.block_count):
self[block_index].get_actives(indiv_material[block_index])
def get_random_input(self, block_material: BlockMaterial, req_dtype, _min=None, _max=None, exclude=[]):
'''
search the genome of the block_material between _min and _max, for a node that outputs the req_dtype.
return None if we failed to find a matching input.
note _max is exclusive so [_min,_max)
'''
ezLogging.debug("%s - Inside get_random_input; req_dtype: %s, _min: %s, _max: %s, exclude: %s" % (block_material.id, req_dtype, _min, _max, exclude))
if _min is None:
_min = -1*self.input_count
if _max is None:
_max = self.main_count
choices = np.arange(_min, _max)
for val in exclude:
choices = np.delete(choices, np.where(choices==val))
if len(choices) == 0:
ezLogging.warning("%s - Eliminated all possible input nodes with exclude: %s" % (block_material.id, exclude))
return None
else:
# exhuastively try each choice to see if we can get datatypes to match
poss_inputs = np.random.choice(a=choices, size=len(choices), replace=False)
for input_index in poss_inputs:
input_dtype = self.get_node_dtype(block_material, input_index, "output")
ezLogging.debug("%s - trying to match index %i with %s to %s" % (block_material.id, input_index, input_dtype, req_dtype))
if req_dtype == input_dtype:
return input_index
else:
pass
# none of the poss_inputs worked, failed to find matching input
ezLogging.warning("%s - None of the input nodes matched for req_dtype: %s, exclude: %s, min: %s, max: %s" % (block_material.id, req_dtype, exclude, _min, _max))
return None
def get_actives(self, block_material: BlockMaterial):
'''
method will go through and set the attributes block_material.active_nodes and active_args.
active_nodes will include all output_nodes, a subset of main_nodes and input_nodes.
'''
ezLogging.info("%s - Inside get_actives" % (block_material.id))
block_material.active_nodes = set(np.arange(self.main_count, self.main_count+self.output_count))
block_material.active_args = set()
#block_material.active_ftns = set()
# add feeds into the output_nodes
for node_input in range(self.main_count, self.main_count+self.output_count):
block_material.active_nodes.update([block_material[node_input]])
for node_index in reversed(range(self.main_count)):
if node_index in block_material.active_nodes:
# then add the input nodes to active list
block_material.active_nodes.update(block_material[node_index]["inputs"])
block_material.active_args.update(block_material[node_index]["args"])
else:
pass
# sort
block_material.active_nodes = sorted(list(block_material.active_nodes))
ezLogging.debug("%s - active nodes: %s" % (block_material.id, block_material.active_nodes))
block_material.active_args = sorted(list(block_material.active_args))
ezLogging.debug("%s - active args: %s" % (block_material.id, block_material.active_args))
def __init__(self, value=None):
if value is None:
self.value = None
self.mutate()
else:
self.value = value
ezLogging.debug("%s-%s - Initialize ArgumentType_TFFilterSize Class to %f" % (None, None, self.value))
def __init__(self):
ezLogging.debug("%s-%s - Initialize BlockArguments_Abstract Class" %
(None, None))
self.arg_count = 0
self.each_type = []
self.each_weight = []
self.arg_types = []
def mutate(self):
import tensorflow as tf
choices = [tf.nn.relu, tf.nn.sigmoid, tf.nn.tanh, tf.nn.elu, None]
if self.value in choices:
choices.remove(self.value) # works in-place
self.value = np.random.choice(choices)
self.get_name()
ezLogging.debug("%s-%s - Mutated ArgumentType_TFActivation to %s" % (None, None, self.name))
def __init__(self, value=None):
if value is None:
self.value = None
self.mutate()
else:
self.value = value
self.get_name()
ezLogging.debug("%s-%s - Initialize ArgumentType_TFActivation Class to %s" % (None, None, self.name))
def mutate(self):
#choices = rnd.random_integers(1, 8)
choices = list(np.arange(1,8+1))
if self.value in choices:
choices.remove(self.value) # works in-place
pow2 = np.random.choice(choices)
self.value = int(2**pow2)
ezLogging.debug("%s-%s - Mutated ArgumentType_Pow2 to %f" % (None, None, self.value))
def __init__(self):
ezLogging.debug(
"%s-%s - Initialize BlockArguments_TransferLearning Class" %
(None, None))
BlockArguments_Abstract.__init__(self)
self.arg_count = 1 * 3
arg_dict = {argument_types.ArgumentType_Int0to25: 1}
self.init_from_weight_dict(arg_dict)
def __init__(self):
ezLogging.debug(
"%s-%s - Initialize BlockArgumentsSmallFloatOnly Class" %
(None, None))
BlockArguments_Abstract.__init__(self)
self.arg_count = 20
arg_dict = {argument_types.ArgumentType_SmallFloats: 1}
self.init_from_weight_dict(arg_dict)
def init_from_weight_dict(self, weight_dict):
'''
like with BlockArguments_Abstract, we have a method to take in a weight_dict that maybe have values equal to 1, and then tools.build_weights will clean up the weights to proper floats between 0 and 1.
'''
ezLogging.debug("%s-%s - Inside init_from_weight_dict" % (None, None))
operators, weights = tools.build_weights(weight_dict)
self.operators = operators
self.weights = weights
def mate(self, parent1: IndividualMaterial, parent2: IndividualMaterial, block_index: int):
'''
wrapper method to call the block's mate definition
'''
ezLogging.info("%s+%s-%s - Sending to Block Mate Definition" % (parent1.id, parent2.id, self.nickname))
children = self.mate_def.mate(parent1, parent2, self, block_index)
ezLogging.debug("%s+%s-%s - Received %i Children from Block Mate Definition" % (parent1.id, parent2.id, self.nickname, len(children)))
return children
def mut_uniform(self):
if self.value == 0:
low = 0
high = 5
else:
low = self.value*.85
high = self.value * 1.15
ezLogging.debug("%s-%s - numpy.random.uniform(%f,%f)" % (None, None, low, high))
self.value = rnd.uniform(low,high)
def mut_normal(self):
if self.value == 0:
mean = 3
std = 3*.1
else:
mean = self.value
std = self.value * .1
ezLogging.debug("%s-%s - numpy.random.normal(%f,%f)" % (None, None, mean, std))
self.value = rnd.normal(mean, std)
def postprocess_block_evaluate(self, block_material):
'''
should always happen after we evaluate. important to blow away block_material.evaluated to clear up memory
can always customize this method which is why we included it in BlockEvaluate and not BlockDefinition
'''
ezLogging.debug("%s - Processing after Evaluation" % (block_material.id))
block_material.evaluated = None
block_material.need_evaluate = False
def __init__(self):
ezLogging.debug(
"%s-%s - Initialize BlockShapeMeta_SymbolicRegression25 Class" %
(None, None))
input_dtypes = [np.ndarray]
output_dtypes = [np.ndarray]
main_count = 25
BlockShapeMeta_Abstract.__init__(self, input_dtypes, output_dtypes,
main_count)
def mutate(self):
roll = rnd.random_integers(0,1)
if roll == 0:
self.mut_normal()
elif roll == 1:
self.mut_uniform()
else:
pass
ezLogging.debug("%s-%s - Mutated ArgumentType_SmallFloats to %f" % (None, None, self.value))
def __init__(self):
ezLogging.debug(
"%s-%s - Initialize BlockShapeMeta_DataAugmentation Class" %
(None, None))
import Augmentor
input_dtypes = [Augmentor.Pipeline]
output_dtypes = [Augmentor.Pipeline]
main_count = 10
super().__init__(input_dtypes, output_dtypes, main_count)
def preprocess_block_evaluate(self, block_material):
'''
should always happen before we evaluate...should be in BlockDefinition.evaluate()
Note we can always customize this to our block needs which is why we included in BlockEvaluate instead of BlockDefinition
'''
ezLogging.debug("%s - Reset for Evaluation" % (block_material.id))
block_material.output = None
block_material.evaluated = [None] * len(block_material.genome)
block_material.dead = False
def __init__(self):
ezLogging.debug(
"%s-%s - Initialize BlockShapeMeta_TransferLearning Class" %
(None, None))
# don't want it imported all the time so we didn't put it at the top of script
import tensorflow as tf
input_dtypes = [tf.keras.layers]
output_dtypes = [tf.keras.layers]
main_count = 1 #has to be one if using BlockEvaluate_TFKeras_TransferLearning2()
super().__init__(input_dtypes, output_dtypes, main_count)
def __init__(self):
ezLogging.debug("%s-%s - Initialize BlockArgumentsSize50 Class" %
(None, None))
BlockArguments_Abstract.__init__(self)
self.arg_count = 50
arg_dict = {
argument_types.ArgumentType_Ints: 1,
argument_types.ArgumentType_Pow2: 1
}
self.init_from_weight_dict(arg_dict)
def __init__(self, block_defs: List[BlockDefinition],
evaluate_def: IndividualEvaluate_Abstract,
mutate_def: IndividualMutate_Abstract,
mate_def: IndividualMate_Abstract):
ezLogging.debug("%s-%s - Starting Initialize Individual" %
(None, None))
self.block_defs = block_defs
self.block_count = len(block_defs)
self.mutate_def = mutate_def()
self.mate_def = mate_def()
self.evaluate_def = evaluate_def()