def runNeuralNetwork(networkArchitecture, addBatchNormalization=False):
writeLog("starting process for: " + networkArchitecture)
train_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
fill_mode='nearest',
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
# img rgb => 3 channels => depth 3
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
# to free memory
if K.backend() == 'tensorflow':
K.clear_session()
learningRate = getLearningRate(networkArchitecture)
model = createModelForNeuralNetwork(networkArchitecture, input_shape,
addBatchNormalization)
model = getBestModel(model, learningRate, train_generator,
validation_generator)
scores = model.evaluate_generator(validation_generator,
nb_validation_samples)
accuracy = scores[1]
writeLog("Accuracy on test data is: " + str(accuracy))
del train_generator
del validation_generator
del train_datagen
del test_datagen
del model
gc.collect()
return scores[1]
def runBests(bests, addBatchNormalization):
for arch in bests:
firstRun = runNeuralNetwork(arch, addBatchNormalization)
# secondRun = runNeuralNetworkCifar(arch, addDropout, addBatchNormalization)
# thirdRun = runNeuralNetworkCifar(arch, addDropout, addBatchNormalization)
# result = (firstRun + secondRun + thirdRun)/3
writeLog(arch + " => addBatchNormalization: " +
str(addBatchNormalization))
writeLog(arch + " => firstRun: " + str(firstRun))
def createModelForNeuralNetwork(networkArchitecture, input_shape,
addBatchNormalization):
layerQuantity = getLayerQuantity(networkArchitecture)
convQuantity = getConvQuant(networkArchitecture)
pool = hasPool(networkArchitecture)
fcQuantity = getFCquantity(networkArchitecture)
addDropout = hasDropout(networkArchitecture)
if (not pool and layerQuantity == 3
and convQuantity == 3): # adicionando pool na maior arquitetura
writeLog("adicionando pool em arquitetura muito grande")
pool = True
model = Sequential()
filterLenght = 32
for layer in range(layerQuantity):
for conv in range(convQuantity):
model.add(
Conv2D(filterLenght, (3, 3),
activation='relu',
padding='same',
input_shape=input_shape))
if (((conv + layer) % 2) == 1):
filterLenght = filterLenght * 2 #duplica o tamanho do filtro a cada duas camadas convolutivas
if addBatchNormalization:
model.add(BatchNormalization())
if pool:
model.add(MaxPooling2D(pool_size=(2, 2)))
if addDropout:
model.add(Dropout(0.25))
model.add(Flatten())
for fc in range(fcQuantity):
model.add(Dense(64))
model.add(Activation('relu'))
if addDropout:
model.add(Dropout(0.5))
if num_classes == 2:
model.add(Dense(1))
model.add(Activation('sigmoid'))
else:
model.add(Dense(num_classes, activation='softmax'))
writeModelSummaryLog(model)
return model
def log_history(history):
acc = "acc = " + str(history.history['acc'])
val_acc = "val_acc = " + str(history.history['val_acc'])
loss = "loss = " + str(history.history['loss'])
val_loss = "val_loss = " + str(history.history['val_loss'])
writeLog(acc)
writeLog(val_acc)
writeLog(loss)
writeLog(val_loss)
return
def print_final_stats():
"""
Prints a final review of the overall evolutionary process.
:return: Nothing.
"""
if hasattr(params['FITNESS_FUNCTION'], "training_test"):
print("\n\nBest:\n Training fitness:\t",
trackers.best_ever.training_fitness)
print(" Test fitness:\t\t", trackers.best_ever.test_fitness)
else:
print("\n\nBest:\n Fitness:\t", trackers.best_ever.fitness)
print(" Phenotype:", trackers.best_ever.phenotype)
writeLog(" Best Phenotype: " + trackers.best_ever.phenotype)
print(" Genome:", trackers.best_ever.genome)
writeLog(" Best Genome: " + str(trackers.best_ever.genome))
print_generation_stats()
def generate_new_genome_and_phenotype():
writeLog('Creating new individual values')
depths = range(params['BNF_GRAMMAR'].min_ramp + 1,
params['MAX_INIT_TREE_DEPTH'] + 1)
size = params['POPULATION_SIZE']
if size < len(depths):
depths = depths[:int(size)]
max_depth = depths[int(len(depths) / 2)]
# Initialise an instance of the tree class
ind_tree = Tree(str(params['BNF_GRAMMAR'].start_rule["symbol"]), None)
# Generate a tree
genome, output, nodes, depth = pi_grow(ind_tree, max_depth)
# Get remaining individual information
phenotype, invalid, used_cod = "".join(output), False, len(genome)
return phenotype, nodes, genome, depth, used_cod, invalid
def search_loop():
"""
This is a standard search process for an evolutionary algorithm. Loop over
a given number of generations.
:return: The final population after the evolutionary process has run for
the specified number of generations.
"""
if params['MULTICORE']:
# initialize pool once, if mutlicore is enabled
params['POOL'] = Pool(processes=params['CORES'],
initializer=pool_init,
initargs=(params, )) # , maxtasksperchild=1)
# Initialise population
individuals = initialisation(params['POPULATION_SIZE'])
# Evaluate initial population
individuals = evaluate_fitness(individuals)
# Generate statistics for run so far
get_stats(individuals)
writeLog("Inicialization...")
for counter, i in enumerate(individuals):
print(str(counter) + " - " + i.__str__())
writeLog(str(counter) + " - " + i.__str__())
# import pdb; pdb.set_trace()
# Traditional GE
for generation in range(1, (params['GENERATIONS'] + 1)):
writeLog("Generation: " + str(generation))
stats['gen'] = generation
# New generation
individuals = params['STEP'](individuals)
for counter, i in enumerate(individuals):
print(str(counter) + " - " + i.__str__())
writeLog(str(counter) + " - " + i.__str__())
# import pdb; pdb.set_trace()
if params['MULTICORE']:
# Close the workers pool (otherwise they'll live on forever).
params['POOL'].close()
return individuals
def search_loop_from_state():
"""
Run the evolutionary search process from a loaded state. Pick up where
it left off previously.
:return: The final population after the evolutionary process has run for
the specified number of generations.
"""
individuals = trackers.state_individuals
writeLog("Inicializing from previous state...")
for counter, i in enumerate(individuals):
print(str(counter) + " - " + i.__str__())
writeLog(str(counter) + " - " + i.__str__())
if params['MULTICORE']:
# initialize pool once, if mutlicore is enabled
params['POOL'] = Pool(processes=params['CORES'],
initializer=pool_init,
initargs=(params, )) # , maxtasksperchild=1)
# Traditional GE
for generation in range(stats['gen'] + 1, (params['GENERATIONS'] + 1)):
writeLog("Generation: " + str(generation))
stats['gen'] = generation
# New generation
individuals = params['STEP'](individuals)
for counter, i in enumerate(individuals):
print(str(counter) + " - " + i.__str__())
writeLog(str(counter) + " - " + i.__str__())
# import pdb; pdb.set_trace()
if params['MULTICORE']:
# Close the workers pool (otherwise they'll live on forever).
params['POOL'].close()
return individuals
def runBests(bests, addDropout, addBatchNormalization, useDataAugmentation):
for arch in bests:
firstRun = runNeuralNetworkCifar(arch, addDropout,
addBatchNormalization,
useDataAugmentation)
secondRun = runNeuralNetworkCifar(arch, addDropout,
addBatchNormalization,
useDataAugmentation)
thirdRun = runNeuralNetworkCifar(arch, addDropout,
addBatchNormalization,
useDataAugmentation)
result = (firstRun + secondRun + thirdRun) / 3
writeLog(arch + " => addDropout: " + str(addDropout) +
"; addBatchNormalization: " + str(addBatchNormalization) +
"; useDataAugmentation: " + str(useDataAugmentation))
writeLog(arch + " => firstRun: " + str(firstRun) + "; secondRun: " +
str(secondRun) + "; thirdRun: " + str(thirdRun))
writeLog(arch + " => media: " + result)
def evaluate(self):
"""
Evaluates phenotype in using the fitness function set in the params
dictionary. For regression/classification problems, allows for
evaluation on either training or test distributions. Sets fitness
value.
:return: Nothing unless multicore evaluation is being used. In that
case, returns self.
"""
# Evaluate fitness using specified fitness function.
# import pdb; pdb.set_trace()
tries = 0
while True:
try:
# import pdb; pdb.set_trace()
if (tries < 5):
self.fitness = runNeuralNetwork(self.phenotype)
else:
writeLog('5 attempts reached for ' + self.phenotype)
self.fitness = 0
break
except Exception as error:
# import pdb; pdb.set_trace()
writeLog('[individual.py] Caught this error: ' + repr(error))
writeLog('tries ' + str(tries))
tries += 1
# generate new individual
phenotype, nodes, genome, depth, used_cod, invalid = generate_new_genome_and_phenotype(
)
self.phenotype, self.nodes, self.genome = phenotype, nodes, genome
self.depth, self.used_codons, self.invalid = depth, used_cod, invalid
# self.fitness = runNeuralNetwork(self.phenotype)#params['FITNESS_FUNCTION'](self)
# import pdb; pdb.set_trace()
if params['MULTICORE']:
return self
def runNeuralNetwork(networkArchitecture, use_step_decay=False):
writeLog("starting neuralNetwork_assuncao process for: " +
networkArchitecture)
# load data
data_all = load_data_from_nii_files(data_paths)
images = transform_list_to_array(data_all)
#print(images.shape)
#mask = load_mask_file(data_mask_path)
# preprocessing
#masked_imgs = apply_mask_to_data_files(data_all, mask)
#images = apply_zscore(masked_imgs)
img_quantity = images.shape[0]
indexes_dis, indexes_con = get_shuffled_index_list(img_quantity)
img_quantity_balanced = indexes_dis.shape[0]
train_indexes, test_indexes = split_train_and_test_index_set(
indexes_dis, indexes_con, img_quantity_balanced)
X_train, X_test = split_data_into_training_and_test_sets(
images, train_indexes, test_indexes)
y_train_as_3D, y_test_as_3D = create_outcome_variables(
labels, train_indexes, test_indexes)
y_train, y_test = reshape_outcome_variables_to_categorical(
y_train_as_3D, y_test_as_3D)
data_shape = tuple(X_train.shape[1:])
k.clear_session()
model = createModelForNeuralNetwork(networkArchitecture, data_shape)
# learning_rate = 1e-5
# adam = Adam(lr=learning_rate)
optimizer = getLearningOptFromNetwork(networkArchitecture)
loss = 'binary_crossentropy'
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
# import pdb; pdb.set_trace() # debug
# train the model
start = time.time()
# Let's test the model:
if (use_step_decay):
# alterar o learning rate em determinados pontos
lrate = LearningRateScheduler(step_decay)
callbacks_list = [lrate]
model_info = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks_list)
else:
model_info = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=epochs,
batch_size=batch_size)
evaluation = model.evaluate(X_test, y_test)
end = time.time()
log_execution_time(start, end)
log_history(model_info)
loss_result = (evaluation[0])
accuracy_result = (evaluation[1] * 100)
writeLog('Loss in Test set: %.02f' % loss_result)
writeLog('Accuracy in Test set: %.02f' % accuracy_result)
#show_activation('conv2d_1', model, X_train)
memory_clean(model)
return accuracy_result
def log_execution_time(start_time, end_time):
diff = int(end_time - start_time)
minutes, seconds = diff // 60, diff % 60
writeLog("Model took minutes to train " + str(minutes) + ':' +
str(seconds).zfill(2))
return
def runNeuralNetworkCifar(networkArchitecture,
addDropout=False,
addBatchNormalization=False,
useDataAugmentation=False):
writeLog("starting process for: " + networkArchitecture)
# load cifar data
(train_features, train_labels), (test_features,
test_labels) = cifar10.load_data()
# split the data on test (test_features_test) and validation (test_features_val)
test_features_val, test_features_test, test_labels_val, test_labels_test = train_test_split(
test_features, test_labels, test_size=0.2, random_state=42)
num_classes = len(np.unique(train_labels))
train_features = train_features.astype("float") / 255.0
test_features_val = test_features_val.astype("float") / 255.0
test_features_test = test_features_test.astype("float") / 255.0
# convert the labels from integers to vectors
lb = LabelBinarizer()
train_labels = lb.fit_transform(train_labels)
test_labels_val = lb.transform(test_labels_val)
test_labels_test = lb.transform(test_labels_test)
input_shape = (32, 32, 3)
batch_size = 128
epochs = 10
# to free memory
if K.backend() == 'tensorflow':
K.clear_session()
# Create the model according to the networkArchitecture
model = createModelForNeuralNetwork(networkArchitecture, input_shape,
num_classes, addDropout,
addBatchNormalization)
# Compile the model
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
# train the model
start = time.time()
if (useDataAugmentation):
# adding data augmentation
datagen = ImageDataGenerator(zoom_range=0.2, horizontal_flip=True)
model_info = model.fit_generator(
datagen.flow(train_features, train_labels, batch_size=batch_size),
steps_per_epoch=train_features.shape[0] // batch_size,
epochs=epochs,
validation_data=(test_features_val, test_labels_val),
verbose=0)
else:
model_info = model.fit(train_features,
train_labels,
batch_size=batch_size,
epochs=epochs,
validation_data=(test_features_val,
test_labels_val),
verbose=0)
end = time.time()
timeMsg = "Model took seconds to train " + str((end - start))
print(timeMsg)
writeLog(timeMsg)
# compute test accuracy
accuracyValue = accuracy(test_features_test, test_labels_test, model)
accMsg = "Accuracy on test data is: " + str(accuracyValue)
print(accMsg)
writeLog(accMsg)
return accuracyValue
# David Fagan, Stefan Forstenlechner,
# and Erik Hemberg
# Hereby licensed under the GNU GPL v3.
""" Python GE implementation """
from utilities.algorithm.general import check_python_version
check_python_version()
from stats.stats import get_stats
from algorithm.parameters import params, set_params
import sys
from writeFileHelper import writeLog
def mane():
""" Run program """
# Run evolution
individuals = params['SEARCH_LOOP']()
# Print final review
get_stats(individuals, end=True)
if __name__ == "__main__":
writeLog('Starting Process....')
set_params(sys.argv[1:]) # exclude the ponyge.py arg itself
mane()
def runNeuralNetwork(networkArchitecture,
data_dir,
epochs=100,
batch_size=32,
img_width=120,
img_height=120):
writeLog("starting neuralNetwork_assuncao_outrasBases process for: " +
networkArchitecture)
#####################################
# basic configuration - dont change
#####################################
input_shape = getInputShape(img_width, img_height)
train_data_dir = data_dir + '/train'
validation_data_dir = data_dir + '/validation'
num_classes = getNumberOfClasses(train_data_dir)
nb_train_samples = getQuantityOfFilesInAFolder(
train_data_dir) # dividido igualmente entre as classes
nb_validation_samples = getQuantityOfFilesInAFolder(
validation_data_dir) # dividido igualmente entre as classes
# print("nb_train_samples: " + str(nb_train_samples) + "; nb_validation_samples: " + str(nb_validation_samples))
# print("num_classes: "+str(num_classes))
#####################################
train_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
fill_mode='nearest',
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
# class_mode='categorical' ???
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
# to free memory
if K.backend() == 'tensorflow':
K.clear_session()
model = createModelForNeuralNetwork(networkArchitecture,
input_shape,
numClasses=num_classes)
optimizer = getLearningOptFromNetwork(networkArchitecture)
model.compile(optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# alterar o learning rate em determinados pontos
lrate = LearningRateScheduler(step_decay)
callbacks_list = [lrate]
# model training
start = time.time()
history = model.fit_generator(
train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size,
callbacks=callbacks_list #[earlyStopping, saveBestModel]
)
writeLog("[INFO] evaluating network...")
scores = model.evaluate_generator(validation_generator,
nb_validation_samples)
end = time.time()
logRunTime(start, end)
logHistoryLog(history)
# writeLog("[INFO] evaluating network...")
# predictions = model.predict(test_features_val, batch_size=batch_size)
accuracy = scores[1]
writeLog("Accuracy on test data is: " + str(accuracy))
memoryClean(train_generator, validation_generator, train_datagen,
test_datagen, model)
return accuracy
def runNeuralNetwork(networkArchitecture,
epochs=400,
batch_size=128,
useDataAugmentation=False):
writeLog("starting neuralNetwork_assuncao process for: " +
networkArchitecture)
# load cifar data
(train_features, train_labels), (test_features,
test_labels) = cifar10.load_data()
# split the data on test (test_features_test) and validation (test_features_val)
test_features_val, test_features_test, test_labels_val, test_labels_test = train_test_split(
test_features, test_labels, test_size=0.2, random_state=42)
train_features = train_features.astype("float") / 255.0
test_features_val = test_features_val.astype("float") / 255.0
test_features_test = test_features_test.astype("float") / 255.0
# convert the labels from integers to vectors
lb = LabelBinarizer()
train_labels = lb.fit_transform(train_labels)
test_labels_val = lb.transform(test_labels_val)
test_labels_test = lb.transform(test_labels_test)
# to free memory
if K.backend() == 'tensorflow':
K.clear_session()
# Create the model according to the networkArchitecture
model = createModelForNeuralNetwork(networkArchitecture, input_shape)
optimizer = getLearningOptFromNetwork(networkArchitecture)
# Compile the model
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
# train the model
start = time.time()
# alterar o learning rate em determinados pontos
lrate = LearningRateScheduler(step_decay)
callbacks_list = [lrate]
if (useDataAugmentation):
# adding data augmentation
datagen = ImageDataGenerator(zoom_range=0.2, horizontal_flip=True)
model_info = model.fit_generator(
datagen.flow(train_features, train_labels, batch_size=batch_size),
steps_per_epoch=train_features.shape[0] // batch_size,
epochs=epochs,
validation_data=(test_features_val, test_labels_val),
callbacks=callbacks_list,
verbose=0)
else:
model_info = model.fit(train_features,
train_labels,
batch_size=batch_size,
epochs=epochs,
validation_data=(test_features_val,
test_labels_val),
callbacks=callbacks_list,
verbose=1)
writeLog("[INFO] evaluating network...")
# predictions = model.predict(test_features_val, batch_size=batch_size)
end = time.time()
logRunTime(start, end)
logHistoryLog(model_info)
# compute test accuracy
accuracyValue = accuracy(test_features_test, test_labels_test, model)
writeLog("Accuracy on test data is: " + str(accuracyValue))
memoryClean(model)
return accuracyValue