def visualize_fcn(self, layers, image_size, canvas_size):
Logger.log("Visualizing layers: " + ", ".join([n[0] for n in layers]))
model = [m for m in self.model.modules()
][1] if params['CUDA_ENABLED'] else self.model
for name, fname in layers:
data = Visualizer.from_torch(model.__getattr__(name).weight.data)
Visualizer.visualize_fcn(data, fname + '.png', image_size,
canvas_size)
def cleanup(self, inds):
best_ind, best_fitness = None, float("inf")
for ind in inds:
if best_fitness > ind.fitness:
best_fitness = ind.fitness
best_ind = ind
best_rnk = ",".join([str(i) for i in ind.stats.getList("rankHist")])
Logger.fwrite("rankHist", "{},{},{}".format(params['CURRENT_GENERATION'], best_fitness, best_rnk))
def print_confusion_matrix(self, pred, y, num_classes):
num_pred = pred.size()[0]
pred, y = pred.view(-1).cpu().numpy(), y.view(-1).cpu().numpy()
def dot(v1, v2):
return np.dot(v1.astype(np.int), v2.astype(np.int))
Logger.log("CM ({} cls): TP\tTN\tFP\tFN\tAccurcay\tTotal: {}".format(num_classes, num_pred))
for i in range(0, num_classes):
p_, y_ = pred==i, y==i
p_neg, y_neg = p_==0, y_==0
tp, tn, fp, fn = dot(p_, y_), dot(p_neg, y_neg), dot(p_, y_neg), dot(p_neg, y_)
num_c = p_.sum()
weighted = tp / num_c + tn / (num_pred - num_c)
Logger.log("Class [{:02d}]: {}\t{}\t{}\t{}\t{:.6f}\t{}".format(i, tp, tn, fp, fn, weighted, num_c))
def visualize_conv(self, layers, fname_prepend):
Logger.log("Visualizing conv layers: " + ", ".join(n[0]
for n in layers))
model = [m for m in self.model.modules()
][1] if params['CUDA_ENABLED'] else self.model
for name, fname in layers:
for p_name, p_params in model.named_parameters():
if p_name.find('.' + name +
'.') >= 0 and p_name.find('weight') >= 0:
ffname = params[
'VISUALIZE_PATH'] + fname + fname_prepend + '.png'
Logger.log("\tWriting image {} to {}....".format(
p_name, ffname))
data = Visualizer.from_torch(p_params.data.clone())
Visualizer.visualize_conv(data, ffname)
def read_cifar(self):
Logger.log("Reading images from {} ...".format(params['DATASET']),
info=False)
X, Y = np.array([]), np.array([])
for num in range(1, 6):
datapath = os.path.join(params['DATASET'],
'data_batch_' + str(num))
with open(datapath, 'rb') as f:
d = pickle.load(f, encoding='bytes')
x, y = d[b'data'], np.asarray(d[b'labels'])
X = np.append(X, x)
Y = np.append(Y, y)
X = X.reshape((-1, 32, 32, 3))
Logger.log("Done reading dataset with {} images...".format(len(X)),
info=False)
return X, Y
def read_csv(self):
Logger.log("Reading images from {0} ...".format(params['DATASET']),
info=False)
X, Y = np.array([]), np.array([])
datapath = os.path.join(params['DATASET'], 'dataset.csv')
with open(datapath) as f:
fnames = csv.reader(f)
next(fnames) #Skip header
for row in fnames:
fname = os.path.join(params['DATASET'], row[0])
img = cv.imread(fname) if os.path.isfile(fname) else print(
fname + " does not exist")
img = cv.resize(img, (80, 80))
X = np.append(X, img)
Y = np.append(Y, int(row[1]))
X = X.reshape(tuple([-1] + list(img.shape)))
Logger.log("Done reading dataset with {0} images...".format(
len(Y)),
info=False)
return X, Y
def evaluate(self, ind, **kwargs):
# ind.phenotype will be a string, including function definitions etc.
# When we exec it, it will create a value XXX_output_XXX, but we exec
# inside an empty dict for safety.
p, d = ind.phenotype, {}
genome, output, invalid, max_depth, nodes = ind.tree.get_tree_info(params['BNF_GRAMMAR'].non_terminals.keys(),[], [])
Logger.log("Depth: {0}\tGenome: {1}".format(max_depth, genome))
# Exec the phenotype.
processed_train = ImageProcessor.process_images(self.X_train, ind.tree)
processed_test = ImageProcessor.process_images(self.X_test, ind.tree)
init_size = ImageProcessor.image.shape[0]*ImageProcessor.image.shape[1]*ImageProcessor.image.shape[2]
train_loss = stats('mse')
test_loss = stats('mse_rnk')
kf = KFold(n_splits=params['CROSS_VALIDATION_SPLIT'])
net = RegressionNet([init_size, 9600, 1200, 1])
fitness, fold = 0, 1
for train_index, val_index in kf.split(processed_train):
X_train, X_val = processed_train[train_index], processed_train[val_index]
y_train, y_val = self.y_train[train_index], self.y_train[val_index]
for epoch in range(1, params['NUM_EPOCHS'] + 1):
net.train(epoch, X_train, y_train, train_loss)
if epoch % 5 == 0:
Logger.log("Epoch {}\tTraining loss (MSE): {:.6f}".format(epoch, train_loss.getLoss('mse')))
net.test(X_val, y_val, test_loss)
fitness += test_loss.getLoss('mse_rnk')
Logger.log("Cross Validation [Fold {}/{}] (MSE/MSE_RNK): {:.6f} {:.6f}".format(fold, kf.get_n_splits(), test_loss.getLoss('mse'), test_loss.getLoss('mse_rnk')))
fold = fold + 1
fitness /= kf.get_n_splits()
ind.stats = test_loss
net.test(processed_test, self.y_test, test_loss)
Logger.log("Generalization Loss (MSE/MSE_RNK): {:.6f} {:.6f}".format(test_loss.getLoss('mse'), test_loss.getLoss('mse_rnk')))
params['CURRENT_EVALUATION'] += 1
return fitness
def search_loop():
"""
This is a standard search process for an evolutionary algorithm. Loop over
a given number of generations.
:return: The final population after the evolutionary process has run for
the specified number of generations.
"""
if params['MULTICORE']:
# initialize pool once, if mutlicore is enabled
params['POOL'] = ThreadPool(
processes=params['CORES'],
initializer=pool_init,
initargs=(params, )) # , maxtasksperchild=1)
Logger.log("Generation 0 starts. Initializing...")
# Initialise population
individuals = initialisation(params['POPULATION_SIZE'])
# Evaluate initial population
individuals = evaluate_fitness(individuals)
# Generate statistics for run so far
get_stats(individuals)
# Cleanup after evaluation
if 'cleanup' in dir(params['FITNESS_FUNCTION']):
params['FITNESS_FUNCTION'].cleanup(individuals)
# Traditional GE
for generation in range(1, (params['GENERATIONS'] + 1)):
stats['gen'] = generation
params['CURRENT_EVALUATION'] = 0
params['CURRENT_GENERATION'] = generation
Logger.log("Generation {0} starts...".format(generation))
# New generation
individuals = params['STEP'](individuals)
# Cleanup after evaluation
if 'cleanup' in dir(params['FITNESS_FUNCTION']):
params['FITNESS_FUNCTION'].cleanup(individuals)
Logger.close_all()
if params['MULTICORE']:
# Close the workers pool (otherwise they'll live on forever).
params['POOL'].close()
return individuals
def __init__(self):
# Initialise base fitness function class.
super().__init__()
# Read images from dataset
X, y = DataReader.read_data(params['DATASET_ID'])
# Train & test split
self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(X, y, test_size=0.33, random_state=42)
# Normalize label
mean, std = np.mean(self.y_train), np.std(self.y_train)
self.y_train = np.divide(np.subtract(self.y_train, mean), std)
self.y_test = np.divide(np.subtract(self.y_test, mean), std)
Logger.log("Training & Test split: {0}/{1} with size {2}".format(len(self.X_train), len(self.X_test), self.X_train[0].shape), info=False)
Logger.log("CUDA ENABLED = {}".format(params['CUDA_ENABLED']), info=False)
Logger.fcreate("rankHist", "log.rank")
def evaluate(self, ind, **kwargs):
# ind.phenotype will be a string, including function definitions etc.
# When we exec it, it will create a value XXX_output_XXX, but we exec
# inside an empty dict for safety.
p, d = ind.phenotype, {}
genome, output, invalid, max_depth, nodes = ind.tree.get_tree_info(
params['BNF_GRAMMAR'].non_terminals.keys(), [], [])
Logger.log("Depth: {0}\tGenome: {1}".format(max_depth, genome))
# Exec the phenotype.
X_test, y_test = self.X_test, self.y_test
image_size = X_test[0].shape
flat_ind, kernel_size = NetworkProcessor.process_network(
ind, image_size)
Logger.log("Individual: {}".format(flat_ind))
Logger.log("New kernel size: {}".format(kernel_size))
new_conv_layers = []
for i, k in enumerate(self.conv_layers):
new_conv_layers.append((k[0], kernel_size[i], k[2], k[3], k[4]))
train_loss = stats('mse')
test_loss = stats('accuracy')
kf = KFold(n_splits=params['CROSS_VALIDATION_SPLIT'])
net = ClassificationNet(self.fcn_layers, new_conv_layers)
fitness, fold = 0, 1
Logger.log("Training Start: ")
# Cross validation
s_time = np.empty((kf.get_n_splits()))
validation_acc = np.empty((kf.get_n_splits()))
test_acc = np.empty((kf.get_n_splits()))
for train_index, val_index in kf.split(self.X_train):
X_train, X_val = self.X_train[train_index], self.X_train[val_index]
y_train, y_val = self.y_train[train_index], self.y_train[val_index]
data_train = DataIterator(X_train, y_train, params['BATCH_SIZE'])
early_ckpt, early_stop, early_crit, epsilon = 20, [], params[
'EARLY_STOP_FREQ'], params['EARLY_STOP_EPSILON']
s_time[fold - 1] = time.time()
# Train model
net.model.reinitialize_params()
for epoch in range(1, params['NUM_EPOCHS'] + 1):
# mini-batch training
for x, y in data_train:
net.train(epoch, x, y, train_loss)
# log training loss
if epoch % params['TRAIN_FREQ'] == 0:
Logger.log("Epoch {} Training loss (NLL): {:.6f}".format(
epoch, train_loss.getLoss('mse')))
# log validation/test loss
if epoch % params['VALIDATION_FREQ'] == 0:
net.test(X_val, y_val, test_loss)
Logger.log(
"Epoch {} Validation loss (NLL/Accuracy): {:.6f} {:.6f}"
.format(epoch, test_loss.getLoss('mse'),
test_loss.getLoss('accuracy')))
net.test(X_test, y_test, test_loss)
Logger.log(
"Epoch {} Test loss (NLL/Accuracy): {:.6f} {:.6f}".
format(epoch, test_loss.getLoss('mse'),
test_loss.getLoss('accuracy')))
# check for early stop
if epoch == early_ckpt:
accuracy = net.test(X_test,
y_test,
test_loss,
print_confusion=True)
early_stop.append(accuracy)
if len(early_stop) > 3:
latest_acc = early_stop[-early_crit:]
latest_acc = np.subtract(latest_acc,
latest_acc[1:] + [0])
if (abs(latest_acc[:-1]) < epsilon).all() == True:
Logger.log(
"Early stopping at epoch {} (latest {} ckpts): {}"
.format(
epoch, early_crit, " ".join([
"{:.4f}".format(x)
for x in early_stop[-early_crit:]
])))
break
early_ckpt = min(early_ckpt + 300, early_ckpt * 2)
# Validate model
net.test(X_val, y_val, test_loss)
validation_acc[fold - 1] = test_loss.getLoss('accuracy')
Logger.log(
"Cross Validation [Fold {}/{}] Validation (NLL/Accuracy): {:.6f} {:.6f}"
.format(fold, kf.get_n_splits(), test_loss.getLoss('mse'),
test_loss.getLoss('accuracy')))
# Test model
net.test(X_test, y_test, test_loss)
test_acc[fold - 1] = test_loss.getLoss('accuracy')
Logger.log(
"Cross Validation [Fold {}/{}] Test (NLL/Accuracy): {:.6f} {:.6f}"
.format(fold, kf.get_n_splits(), test_loss.getLoss('mse'),
test_loss.getLoss('accuracy')))
# Calculate time
s_time[fold - 1] = time.time() - s_time[fold - 1]
Logger.log(
"Cross Validation [Fold {}/{}] Training Time (m / m per epoch): {:.3f} {:.3f}"
.format(fold, kf.get_n_splits(), s_time[fold - 1] / 60,
s_time[fold - 1] / 60 / epoch))
fold = fold + 1
fitness = validation_acc.mean()
for i in range(0, kf.get_n_splits()):
Logger.log(
"STAT -- Model[{}/{}] #{:.3f}m Validation / Generalization accuracy (%): {:.4f} {:.4f}"
.format(i, kf.get_n_splits(), s_time[i] / 60,
validation_acc[i] * 100, test_acc[i] * 100))
Logger.log(
"STAT -- Mean Validation / Generatlization accuracy (%): {:.4f} {:.4f}"
.format(validation_acc.mean() * 100,
test_acc.mean() * 100))
# ind.net = net
params['CURRENT_EVALUATION'] += 1
return fitness
def __init__(self):
# Initialise base fitness function class.
super().__init__()
self.fcn_layers = params['FCN_LAYERS']
self.conv_layers = params['CONV_LAYERS']
self.resize = params['RESIZE']
# Read images from dataset
X, y = DataReader.read_data(params['DATASET_ID'])
# Train & test split
self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
X, y, test_size=0.33, random_state=42)
if params['NORMALIZE_LABEL']:
Logger.log("Normalizing labels...")
self.y_train, mean, std = ImageProcessor.normalize(self.y_train)
self.y_test, _, _ = ImageProcessor.normalize(self.y_test,
mean=mean,
std=std)
Logger.log(
"Mean / Std of training set (by channel): {} / {}".format(
mean, std))
# Check class balance between splits
classes, class_balance_train, class_balance_test = check_class_balance(
self.y_train, self.y_test)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("Class Balance --", info=False)
Logger.log("\tClass: \t{}".format("\t".join([str(c)
for c in classes])),
info=False)
Logger.log("\tTrain: \t{}".format("\t".join(
[str(n) for n in class_balance_train])),
info=False)
Logger.log("\tTest: \t{}".format("\t".join(
[str(n) for n in class_balance_test])),
info=False)
Logger.log("\tTotal: \t{}\t{}".format(
"\t".join(
[str(n) for n in class_balance_train + class_balance_test]),
(class_balance_train + class_balance_test).sum()),
info=False)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("General Setup --", info=False)
Logger.log("\tCUDA enabled: \t{}".format(params['CUDA_ENABLED']),
info=False)
Logger.log("\tDebug network enabled: \t{}".format(params['DEBUG_NET']),
info=False)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("Data Preprocess --", info=False)
Logger.log("\tNumber of samples: \t{}".format(len(X)), info=False)
Logger.log("\tTraining / Test split: \t{}/{}".format(
len(self.X_train), len(self.X_test)),
info=False)
Logger.log("\tImage size: \t{}".format(self.X_train[0].shape),
info=False)
Logger.log("\tNormalize label: \t{}".format(params['NORMALIZE_LABEL']),
info=False)
Logger.log("\tNormalize after preprocessing: \t{}".format(
params['NORMALIZE']),
info=False)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("GP Setup --", info=False)
Logger.log("\tGrammar file: \t{}".format(params['GRAMMAR_FILE']),
info=False)
Logger.log("\tPoupulation size: \t{}".format(
params['POPULATION_SIZE']),
info=False)
Logger.log("\tGeneration num: \t{}".format(params['GENERATIONS']),
info=False)
Logger.log("\tImage resizing (after proc): \t{}".format(self.resize),
info=False)
Logger.log("\tTree depth init (Min/Max): \t{}/{}".format(
params['MIN_INIT_TREE_DEPTH'], params['MAX_INIT_TREE_DEPTH']),
info=False)
Logger.log("\tTree depth Max: \t\t{}".format(params['MAX_TREE_DEPTH']),
info=False)
def evaluate(self, ind, **kwargs):
# ind.phenotype will be a string, including function definitions etc.
# When we exec it, it will create a value XXX_output_XXX, but we exec
# inside an empty dict for safety.
p, d = ind.phenotype, {}
genome, output, invalid, max_depth, nodes = ind.tree.get_tree_info(
params['BNF_GRAMMAR'].non_terminals.keys(), [], [])
Logger.log("Depth: {0}\tGenome: {1}".format(max_depth, genome))
ind.tree.print_tree()
## Evolve image preprocessor
Logger.log("Processing Pipeline Start: {} images...".format(
len(self.X_train) + len(self.X_test)))
processed_train = ImageProcessor.process_images(self.X_train,
ind.tree.children[0],
resize=self.resize)
processed_test = ImageProcessor.process_images(self.X_test,
ind.tree.children[0],
resize=self.resize)
# Normalize image by channel
if params['NORMALIZE']:
Logger.log("Normalizing processed images...")
processed_train, mean, std = ImageProcessor.normalize_img(
processed_train)
processed_test, _, _ = ImageProcessor.normalize_img(processed_test,
mean=mean,
std=std)
Logger.log(
"Mean / Std of training set (by channel): {} / {}".format(
mean, std))
# Setup test images
X_test, y_test = processed_test, self.y_test
image = ImageProcessor.image
init_size = image.shape[0] * image.shape[1] * image.shape[2]
## Evolve network structure
if params['EVOLVE_NETWORK']:
Logger.log("Network Structure Selection Start: ")
flat_ind, new_conv_layers = NetworkProcessor.process_network(
ind.tree.children[1], image.shape,
copy.deepcopy(self.conv_layers))
Logger.log("\tIndividual: {}".format(flat_ind))
else:
new_conv_layers = copy.deepcopy(self.conv_layers)
conv_outputs = Network.calc_conv_output(new_conv_layers, image.shape)
Logger.log("\tNew convolution layers: ")
for i, a, b in zip(range(len(new_conv_layers)), new_conv_layers,
conv_outputs):
Logger.log("\tConv / output at layer {}: {}\t=> {}".format(
i, a, b))
# Modify fully connected input size
new_fcn_layers, conv_output = copy.deepcopy(
self.fcn_layers), conv_outputs[-1]
new_fcn_layers[0] = conv_output[0] * conv_output[1] * conv_output[2]
net = eval(params['NETWORK'])(new_fcn_layers, new_conv_layers)
if params['PRETRAIN']:
Logger.log("Using pretrained model at {}".format(
params['PRETRAINED_MODEL']))
net.test(X_test, y_test)
Logger.log("Pretrained initial scores: {}".format(
net.get_test_loss_str()))
def visualize(tag):
if params['VISUALIZE']:
net.visualize_conv([('alex1', 'alex1')], '.' + tag)
kf = KFold(n_splits=params['CROSS_VALIDATION_SPLIT'])
fitness, fold = 0, 1
Logger.log("Training Start: ")
# Cross validation
s_time = np.empty((kf.get_n_splits()))
for train_index, val_index in kf.split(processed_train):
X_train, X_val = processed_train[train_index], processed_train[
val_index]
y_train, y_val = self.y_train[train_index], self.y_train[val_index]
data_train = DataIterator(X_train, y_train, params['BATCH_SIZE'])
early_ckpt, early_stop, early_crit, epsilon = 4, [], params[
'EARLY_STOP_FREQ'], params['EARLY_STOP_EPSILON']
s_time[fold - 1] = time.time()
# Train model
net.model.reinitialize_params()
for epoch in range(1, params['NUM_EPOCHS'] + 1):
# mini-batch training
for x, y in data_train:
net.train(epoch, x, y)
# log training loss
if epoch % params['TRAIN_FREQ'] == 0:
Logger.log("Epoch {} Training loss (NLL): {:.6f}".format(
epoch, net.train_loss.getLoss()))
# log validation/test loss
if epoch % params['VALIDATION_FREQ'] == 0 or epoch < 15:
net.test(X_val, y_val)
Logger.log(
"Epoch {} Validation loss (NLL/Accuracy): {}".format(
epoch, net.get_test_loss_str()))
net.test(X_test, y_test)
Logger.log("Epoch {} Test loss (NLL/Accuracy): {}".format(
epoch, net.get_test_loss_str()))
if epoch % params['VALIDATION_FREQ'] == 0 or epoch < 15:
visualize('g{}_e{}_f{}_ep{}'.format(
params['CURRENT_GENERATION'],
params['CURRENT_EVALUATION'], fold, epoch))
# check for early stop
if epoch == early_ckpt:
accuracy = net.test(X_test, y_test, print_confusion=True)
early_stop.append(accuracy)
if len(early_stop) > 3:
latest_acc = early_stop[-early_crit:]
latest_acc = np.subtract(latest_acc,
latest_acc[1:] + [0])
if (abs(latest_acc[:-1]) < epsilon).all() == True:
Logger.log(
"Early stopping at epoch {} (latest {} ckpts): {}"
.format(
epoch, early_crit, " ".join([
"{:.4f}".format(x)
for x in early_stop[-early_crit:]
])))
break
# early_ckpt = min(early_ckpt+300, early_ckpt*2)
early_ckpt += params['VALIDATION_FREQ']
# Validate model
net.test(X_val, y_val)
net.save_validation_loss()
Logger.log(
"Cross Validation [Fold {}/{}] Validation (NLL/Accuracy): {}".
format(fold, kf.get_n_splits(), net.get_test_loss_str()))
# Test model
net.test(processed_test, self.y_test)
net.save_test_loss()
Logger.log(
"Cross Validation [Fold {}/{}] Test (NLL/Accuracy): {}".format(
fold, kf.get_n_splits(), net.get_test_loss_str()))
# Calculate time
s_time[fold - 1] = time.time() - s_time[fold - 1]
Logger.log(
"Cross Validation [Fold {}/{}] Training Time (m / m per epoch): {:.3f} {:.3f}"
.format(fold, kf.get_n_splits(), s_time[fold - 1] / 60,
s_time[fold - 1] / 60 / epoch))
visualize('g{}_e{}_f{}_ep{}.end'.format(
params['CURRENT_GENERATION'], params['CURRENT_EVALUATION'],
fold, epoch))
fold = fold + 1
fitness = net.get_fitness()
val_log, test_log = np.array(net.validation_log), np.array(
net.test_log)
val_mean, test_mean = val_log.mean(axis=0), test_log.mean(axis=0)
for i, (v, t) in enumerate(zip(val_log, test_log)):
log = " ".join([
"{:.4f} {:.4f}".format(v[idx], t[idx]) for idx in range(len(v))
])
Logger.log(
"STAT -- Model[{}/{}] #{:.3f}m Validation / Generalization: {}"
.format(i, kf.get_n_splits(), s_time[i] / 60, log))
Logger.log("STAT -- Mean Validation / Generatlization: {}".format(
" ".join([
"{:.4f} {:.4f}".format(i, j)
for i, j in zip(val_mean, test_mean)
])))
# ind.net = net
params['CURRENT_EVALUATION'] += 1
return fitness
def log_model(self):
Logger.log("---------------------------------------------------", info=False)
Logger.log("Neural Network Setup --", info=False)
Logger.log("\tEpochs / CV fold: \t{} * {} ({} total)".format(params['NUM_EPOCHS'], params['CROSS_VALIDATION_SPLIT'], params['NUM_EPOCHS']*params['CROSS_VALIDATION_SPLIT']), info=False)
Logger.log("\tBatch size: \t\t{}".format(params['BATCH_SIZE']), info=False)
Logger.log("\tLearning rate / Momentum: \t{} / {}".format(params['LEARNING_RATE'], params['MOMENTUM']), info=False)
Logger.log("\tNetwork structure = \n{}".format(self.model), info=False)
Logger.log("---------------------------------------------------", info=False)
def __init__(self, fcn_layers, conv_layers):
super(EvoPretrainedClassificationNet,
self).__init__(fcn_layers, conv_layers)
pretrained_model_path = params['PRETRAINED_MODEL']
prenet = ClassificationNet(params['FCN_LAYERS'],
params['CONV_LAYERS'],
input_channel=3)
if not os.path.isfile(pretrained_model_path):
Logger.log("Training Pretrained Network: ")
Logger.log("Saving Pretrained Network to: {}".format(
pretrained_model_path))
# torch.save(prenet.model.state_dict(), pretrained_model_path)
prenet.model.load_state_dict(torch.load(pretrained_model_path))
self.model = ConvModel(fcn_layers=fcn_layers, conv_layers=conv_layers)
Logger.log('Transfer parameters start...')
# Transfer the weights
prenet_m = iter(prenet.model.named_parameters())
p_name, p_params = next(prenet_m)
for q_name, q_params in self.model.named_parameters():
if p_name == q_name:
Logger.log('Set parameters: {}'.format(p_name))
# Skip initialize if 1D bias
if p_name.find('bias') < 0:
nn.init.xavier_uniform_(q_params, gain=np.sqrt(2))
# Set extra parameters to zero for debugging
if params['DEBUG_NET']:
Logger.log('Debug network :: Fill 0')
q_params.data.fill_(0)
# Transfer convolution params
if p_name.find('conv') >= 0 and p_name.find('weight') >= 0:
q_params.data[:p_params.data.shape[0], :p_params.data.
shape[1], :, :] = p_params.data.clone()
# template = p_params.data.view((-1, p_params.data.shape[2], p_params.data.shape[3]))
# for sub in patch[:, p_params.data.shape[1]:]:
# sub = template[torch.randperm(len(template))]
# for sub in patch[p_params.data.shape[0]:,:]:
# sub = template[torch.randperm(len(template))]
# Transfer fully connected params
elif p_name.find('fcn') >= 0 and p_name.find('weight') >= 0:
q_params.data[:p_params.data.shape[0], :p_params.data.
shape[1]] = p_params.data.clone()
# Transfer bias params
elif p_name.find('bias') >= 0:
q_params.data[:p_params.data.
shape[0]] = p_params.data.clone()
# with torch.no_grad():
# q_params.copy_(patch)
# Set to next layer of pretrained
try:
p_name, p_params = next(prenet_m)
except StopIteration:
break
else:
print("WARNING: Module {0} is not initialized.".format(q_name))
for q_name, q_params in self.model.named_parameters():
print("Model parameters: ", q_name)
for p_name, p_params in prenet.model.named_parameters():
print("Prenet parameters: ", p_name)
# Release pretrained model
prenet.model.cpu()
del prenet
torch.cuda.empty_cache()
self.optimizer = optim.SGD(self.model.parameters(),
lr=params['LEARNING_RATE'],
momentum=params['MOMENTUM'])
def evaluate(self, ind, **kwargs):
# ind.phenotype will be a string, including function definitions etc.
# When we exec it, it will create a value XXX_output_XXX, but we exec
# inside an empty dict for safety.
p, d = ind.phenotype, {}
genome, output, invalid, max_depth, nodes = ind.tree.get_tree_info(params['BNF_GRAMMAR'].non_terminals.keys(),[], [])
Logger.log("Depth: {0}\tGenome: {1}".format(max_depth, genome))
# Exec the phenotype.
Logger.log("Processing Pipeline Start: {} images...".format(len(self.X_train)+len(self.X_test)))
processed_train = ImageProcessor.process_images(self.X_train, ind, resize=self.resize)
processed_test = ImageProcessor.process_images(self.X_test, ind, resize=self.resize)
image = ImageProcessor.image
init_size = image.shape[0]*image.shape[1]*image.shape[2]
train_loss = stats('mse')
test_loss = stats('accuracy')
kf, freq = KFold(n_splits=params['CROSS_VALIDATION_SPLIT']), params["EPOCH_FREQ"]
net = ClassificationNet(self.layers)
fitness, early_stop, fold = 0, 0, 1
Logger.log("Training Start: ")
for train_index, val_index in kf.split(processed_train):
X_train, X_val = processed_train[train_index], processed_train[val_index]
y_train, y_val = self.y_train[train_index], self.y_train[val_index]
data_train = DataIterator(X_train, y_train, params['BATCH_SIZE'])
prev, early_stop = 0, 0
for epoch in range(1, params['NUM_EPOCHS'] + 1):
batch = 0
for x, y in data_train:
net.train(epoch, x, y, train_loss)
batch += 1
# if batch % 10 == 0:
# Logger.log("Batch {}/{}".format(batch, data_train.num_splits))
if epoch % freq == 0:
Logger.log("Epoch {}\tTraining loss (NLL): {:.6f}".format(epoch, train_loss.getLoss('mse')))
if abs(prev - train_loss.getLoss('mse')) < 1e-6:
early_stop += 1
if early_stop > 10:
Logger.log("Early stopping at epoch {}".format(epoch))
break
else:
early_stop = 0
prev = train_loss.getLoss('mse')
if epoch % freq == 0:
net.test(X_val, y_val, test_loss)
Logger.log("Epoch {}\tTest loss (NLL): {:.6f} {:.6f}".format(epoch, test_loss.getLoss('mse'), test_loss.getLoss('accuracy')))
net.test(X_val, y_val, test_loss)
fitness += test_loss.getLoss('accuracy')
Logger.log("Cross Validation [Fold {}/{}] (MSE/Accuracy): {:.6f} {:.6f}".format(fold, kf.get_n_splits(), test_loss.getLoss('mse'), test_loss.getLoss('accuracy')))
fold = fold + 1
fitness /= kf.get_n_splits()
net.test(processed_test, self.y_test, test_loss)
#ind.net = net
Logger.log("Generalization Loss (MSE/Accuracy): {:.6f} {:.6f}".format(test_loss.getLoss('mse'), test_loss.getLoss('accuracy')))
params['CURRENT_EVALUATION'] += 1
return fitness
def __init__(self):
# Initialise base fitness function class.
super().__init__()
self.layers = [12544, 6272, 10]
self.resize = params['RESIZE']
# Read images from dataset
Logger.log("Reading images from {0} ...".format(params['DATASET']), info=False)
self.X_train = self.read_images('train-images-idx3-ubyte', 60000)
self.y_train = self.read_labels('train-labels-idx1-ubyte', 60000)
Logger.log("Done reading dataset with {0} images...".format(len(self.X_train)), info=False)
# Train & test split
#self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(X, Y, test_size=0.33, random_state=42)
self.X_test = self.read_images('t10k-images-idx3-ubyte', 10000)
self.y_test = self.read_labels('t10k-labels-idx1-ubyte', 10000)
Logger.log("Training & Test split: {0}/{1} with size {2}".format(len(self.X_train), len(self.X_test), self.X_train[0].shape), info=False)
Logger.log("CUDA ENABLED = {}".format(params['CUDA_ENABLED']), info=False)
Logger.log("Using grammar = {}".format(params['GRAMMAR_FILE']), info=False)
Logger.log("Resizing after processing = {}".format(self.resize), info=False)
Logger.log("Batch size = {}".format(params['BATCH_SIZE']), info=False)
Logger.log("Network structure = \n{}".format(ClassificationNet(self.layers).model), info=False)
def __init__(self):
# Initialise base fitness function class.
super().__init__()
self.fcn_layers = params['FCN_LAYERS']
self.conv_layers = params['CONV_LAYERS']
self.resize = params['RESIZE']
# Read images from dataset
Logger.log("Reading images from {} ...".format(params['DATASET']),
info=False)
X, Y = np.array([]), np.array([])
for num in range(1, 6):
datapath = os.path.join(params['DATASET'],
'data_batch_' + str(num))
x, y = read_cifar(datapath)
X = np.append(X, x)
Y = np.append(Y, y)
X = np.reshape(X, (-1, 32, 32, 3))
Logger.log("Done reading dataset with {} images...".format(len(X)),
info=False)
if params['AUGMENT_CHANNEL']:
Logger.log("Augmenting images with extra channel: ", info=False)
_X = np.zeros((len(X), 32, 32, 4))
for idx, src in enumerate(X):
src = np.uint8(src)
aug = cv.cvtColor(cv.pyrMeanShiftFiltering(src, 8, 64),
cv.COLOR_RGB2GRAY)
# aug = np.array([255]*32*32*1, dtype=np.uint8).reshape((32,32,1))
_X[idx] = cv.merge(
tuple([src[:, :, 0], src[:, :, 1], src[:, :, 2], aug]))
params['INPUT_CHANNEL'] = 4
X = _X
Network.assert_net(self.conv_layers, self.fcn_layers,
X[0].shape if not self.resize else self.resize)
# Train & test split
self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
X, Y, test_size=0.33, random_state=42)
if params['NORMALIZE']:
Logger.log("Normalizing processed images...", info=False)
self.X_train, mean, std = ImageProcessor.normalize(self.X_train)
self.X_test, _, _ = ImageProcessor.normalize(self.X_test,
mean=mean,
std=std)
Logger.log(
"Mean / Std of training set (by channel): {} / {}".format(
mean, std),
info=False)
# Check class balance between splits
classes, class_balance_train, class_balance_test = check_class_balance(
self.y_train, self.y_test)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("Class Balance --", info=False)
Logger.log("\tClass: \t{}".format("\t".join([str(c)
for c in classes])),
info=False)
Logger.log("\tTrain: \t{}".format("\t".join(
[str(n) for n in class_balance_train])),
info=False)
Logger.log("\tTest: \t{}".format("\t".join(
[str(n) for n in class_balance_test])),
info=False)
Logger.log("\tTotal: \t{}\t{}".format(
"\t".join(
[str(n) for n in class_balance_train + class_balance_test]),
(class_balance_train + class_balance_test).sum()),
info=False)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("General Setup --", info=False)
Logger.log("\tCUDA enabled: \t{}".format(params['CUDA_ENABLED']),
info=False)
Logger.log("\tDebug network enabled: \t{}".format(params['DEBUG_NET']),
info=False)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("Data Preprocess --", info=False)
Logger.log("\tNumber of samples: \t{}".format(len(X)), info=False)
Logger.log("\tTraining / Test split: \t{}/{}".format(
len(self.X_train), len(self.X_test)),
info=False)
Logger.log("\tImage size: \t{}".format(self.X_train[0].shape),
info=False)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("GP Setup --", info=False)
Logger.log("\tGrammar file: \t{}".format(params['GRAMMAR_FILE']),
info=False)
Logger.log("\tPoupulation size: \t{}".format(
params['POPULATION_SIZE']),
info=False)
Logger.log("\tGeneration num: \t{}".format(params['GENERATIONS']),
info=False)
Logger.log("\tImage resizing (after proc): \t{}".format(self.resize),
info=False)
Logger.log("\tTree depth init (Min/Max): \t{}/{}".format(
params['MIN_INIT_TREE_DEPTH'], params['MAX_INIT_TREE_DEPTH']),
info=False)
Logger.log("\tTree depth Max: \t\t{}".format(params['MAX_TREE_DEPTH']),
info=False)
Logger.log("---------------------------------------------------",
info=False)
Logger.log("Initial Neural Network Setup --", info=False)
Logger.log("\tEpochs / CV fold: \t{} * {} ({} total)".format(
params['NUM_EPOCHS'], params['CROSS_VALIDATION_SPLIT'],
params['NUM_EPOCHS'] * params['CROSS_VALIDATION_SPLIT']),
info=False)
Logger.log("\tBatch size: \t\t{}".format(params['BATCH_SIZE']),
info=False)
Logger.log("\tLearning rate / Momentum: \t{} / {}".format(
params['LEARNING_RATE'], params['MOMENTUM']),
info=False)
Logger.log("\tNetwork structure: \n{}".format(
EvoClassificationNet(self.fcn_layers, self.conv_layers).model),
info=False)