def insertion(child, _, acceptable_fitness):
""" Insertion mutation - never insert after output codon """
if _DEBUG:
printlog('Insertion <', child)
child_len = len(child)
if child_len == 0:
return child
# If first codon is an input codon, never insert before it (since this will
# almost always create an invalid genome).
start_offset = 1 if child[0] in INPUTS else 0
insert_position = start_offset if child_len == 1 else random.randrange(
start_offset, child_len)
codon = random_codon(acceptable_fitness)
child.insert(insert_position, codon)
if _DEBUG:
printlog('Insertion >', child)
return child
def regularize(child):
""" Rearrange modifier codons into consistent order """
def crispr(codon):
""" cut up a codon for sort purposes """
codon = codon.split('_')[1:]
codon = [(p[0], int(p[1:])) for p in reversed(codon)]
return codon
shuffled = True
if _DEBUG:
printlog('Regularize <', child)
while shuffled:
shuffled = False
for i in range(len(child) - 1):
if child[i] in MODIFIERS and child[i + 1] in MODIFIERS:
# Currently depends on the modifier parameter codes being
# in string sort order, but since we only have one of them
# (r) this is not an issue.
if crispr(child[i]) > crispr(child[i + 1]):
child[i], child[i + 1] = child[i + 1], child[i]
shuffled = True
if _DEBUG:
printlog('Regularize >', child)
return child
def inversion(child, _, __):
""" Invert two adjacent codons """
# Cannot invert if 3 or fewer elements
if _DEBUG:
printlog('Inversion <', child)
if len(child) <= 2:
return child
locus = 0 if len(child) == 2 else random.randrange(len(child) - 2)
# Cannot invert if input or output codon
if child[locus] in INPUTS or child[locus + 1] in OUTPUTS:
return child
# Swap codons
child[locus], child[locus + 1] = child[locus + 1], child[locus]
if _DEBUG:
printlog('Inversion >', child)
return child
def on_epoch_end(self, epoch, logs=None):
# Get current LR
if self.learning_rate != self.optimizer.lr.get_value():
self.learning_rate = self.optimizer.lr.get_value()
self.optimizer.clipvalue.set_value(self.theta / self.learning_rate)
if self.verbose:
printlog('Changed Gradient Clipping Value: %f' %
(self.theta / self.learning_rate))
def evaluate(self):
""" Evaluate the model on self.evaluation_path """
printlog('Validating %s model' % self.name)
results = self.model.evaluate_generator(
self.config.evaluation_data_generator(),
steps=self.config.eval_images_count() // self.config.batch_size)
print("Loss = %.5f, PeekSignalToNoiseRatio = %.5f" %
(results[0], results[1]))
def conjugation(child, conjugate, _):
""" Conjugate two genomes - always at least one codon from each contributor """
if _DEBUG:
printlog('Conjugation <', child)
splice = random.randrange(1, len(child))
child = child[:-splice] + conjugate[-splice:]
if _DEBUG:
printlog('Conjugation >', child)
return child
def random_codon(acceptable_fitness):
""" Choose a random codon from PRIMORDIAL_CODONS, with suitable fitness.
Will always return because of the random diceroll.
"""
# Try to get a codon of acceptable fitness but give up if we have been
# looking too long.
for _ in range(0, 100):
codon = random.choice(PRIMORDIAL_CODONS)
if acceptable_fitness(codon):
if _DEBUG:
printlog('Random Codon =', codon)
break
return codon
def on_epoch_end(self, epoch, logs=None):
self.state['epoch_count'] += 1
# Currently, for everything we track, lower is better.
for k in logs:
if k not in self.state[
'best_values'] or logs[k] > self.state['best_values'][k]:
self.state['best_values'][k] = float(logs[k])
self.state['best_epoch'][k] = self.state['epoch_count']
with open(self.path, 'w') as jsonfile:
json.dump(self.state, jsonfile, indent=4)
if self.verbose:
printlog('Completed epoch', self.state['epoch_count'])
def __init__(self, name, config, loss_function='PeakSignaltoNoiseRatio'):
self.name = name
self.config = copy.deepcopy(config)
self.loss_function = loss_function
self.val_loss_function = 'val_' + loss_function
self.evaluation_function = LOSS_FUNCTIONS[loss_function](config.border)
if os.path.isfile(config.paths['model']):
if self.config.verbose:
printlog('Loading existing .h5 model: ' +
config.paths['model'])
self.model = load_model(
config.paths['model'],
custom_objects={loss_function: self.evaluation_function})
else:
if self.config.verbose:
printlog('Creating new untrained model')
self.model = self.create_model(load_weights=False)
def transposition(child, _, __):
""" Transposition mutation - never transpose output or input codon """
if _DEBUG:
printlog('Transposition <', child)
# Cannot transpose if 2 or fewer elements
if len(child) <= 2:
return child
splice = random.randrange(len(child) - 1)
codon = child[splice]
if codon in INPUTS or codon in OUTPUTS:
return child
del child[splice]
splice = random.randrange(len(child) - 1)
child.insert(splice, codon)
if _DEBUG:
printlog(' Transposed :', codon)
printlog('Transposition >', child)
return child
def deletion(child, _, min_len):
""" Deletion mutation - never delete output codon or conserved codons! """
if _DEBUG:
printlog('Deletion <', child)
child_len = len(child)
if child_len <= min_len:
return child
dpos = random.randrange(child_len - 1)
if child[dpos] in CONSERVED_CODONS:
return child
del child[dpos]
if _DEBUG:
printlog('Deletion <', child)
return child
def __init__(self, config):
super().__init__()
self.verbose = config.verbose
self.path = config.paths['state']
if os.path.isfile(self.path):
if config.verbose:
printlog('Loading existing .json state: ' + self.path)
with open(self.path, 'r') as jsonfile:
self.state = json.load(jsonfile)
# Update state with current run information
self.state['config'] = config.config
else:
printlog('Initializing new model')
self.state = {
'epoch_count': 0,
'best_values': {},
'best_epoch': {},
'config': config.config
}
def test_printlog(capsys):
""" Test for misc.printlog """
# Unformatted log print
out, err = capsys.readouterr()
misc.printlog(_T1)
out, err = capsys.readouterr()
assert err == ''
outs = out.split(': ')
assert len(outs) == 2
assert outs[1] == _T1_N
# Formatted log print
out, err = capsys.readouterr()
misc.printlog(_T1, _T2)
out, err = capsys.readouterr()
outs = out.split(': ')
assert len(outs) == 2
assert outs[1] == _T1 + ' ' + _T2_N
def tesselate_pair(alpha_paths, beta_paths, config):
""" This version tesellates matched pairs of images, with identical shuffling behavior. Used for model training """
# Convert non-lists to lists
alpha_paths = alpha_paths if isinstance(
alpha_paths, (list, tuple)) else [alpha_paths]
beta_paths = beta_paths if isinstance(
beta_paths, (list, tuple)) else [beta_paths]
all_paths = list(zip(alpha_paths, beta_paths))
# Shuffle the lists
if config.shuffle:
random.shuffle(all_paths)
# Process the image file pairs
for alpha_path, beta_path in all_paths:
# Extract the tiles from paired image files. One gotcha to keep in mind - there will be a
# great disturbance in the force if the alpha tiles get cached but the beta tiles don't,
# AND we happen to be messing with the tiles because of quality < 1.0. So we make sure
# that only the read of the beta tiles can turn off the cache (which happens after a
# cache store)
alpha_tiles = extract_tiles(alpha_path, config, can_disable=False)
beta_tiles = extract_tiles(beta_path, config, can_disable=True)
if len(alpha_tiles) != len(beta_tiles):
printlog('Tesselation error: file pairs {} and {} have different tile counts {} and {}'.format(
alpha_path, beta_path, len(alpha_tiles), len(beta_tiles)))
elif alpha_tiles:
# If we are doing quality selection, then we need to fix the cache the first time
# through. The trick, however, is that we need to use the quality ratings for
# the beta tiles to determine the order of the alpha tiles, so things remain
# properly paired.
if config.quality < 1.0 and beta_path not in CACHED_QUALITY:
beta_tiles, beta_indexes = update_cache_quality(
beta_path, beta_tiles, config.quality)
alpha_tiles, _ = update_cache_quality(
alpha_path, alpha_tiles, config.quality, beta_indexes)
# Shuffle tiles
tiles_list = list(range(len(alpha_tiles)))
if config.shuffle:
random.shuffle(tiles_list)
# Generate tiles
skip_count = random.randint(0, 4) if config.skip else 0
for tile_index in tiles_list:
if skip_count <= 0:
skip_count = random.randint(0, 4) if config.skip else 0
# GPU : PU please check this
if config.residual: # returns the difference between beta-alpha
yield (alpha_tiles[tile_index], beta_tiles[tile_index] - alpha_tiles[tile_index])
else:
yield (alpha_tiles[tile_index], beta_tiles[tile_index])
else:
skip_count -= 1
def mutate(parent,
conjugate,
tried,
min_len=2,
max_len=90,
odds=(4, 8, 9, 10, 11, 12),
best_fitness=0.0,
statistics=None,
viable=None):
""" Mutate a model. There are 6 possible mutations that can occur:
point point mutation of a parameter of a codon
insert insert a new codon
delete delete a codon
invert two adjacent codons are flipped
transpose move a codon somewhere else in the genome
conjugate replace codons with codons in another genome
Parameters:
parent parental genome (list of codons; if string will be converted)
conjugate parental genome (only used for conjugation)
tried genomes we have already tried (for quick rejection; = parents + graveyard)
min_len minimum length of the resulting genome
max_len maximum length of the resulting genome
odds list of *cumulative* odds of point, insert, delete, transpose, conjugate
best_fitness the fitness of the best genome found so far.
statistics dictionary of codon statistics for guiding evolotion
viable viability function; takes a codon list, returns true if it is acceptable
"""
if KERNELS is None:
printlog('Fatal Error: configure_environment was not called!')
exit(1)
if not isinstance(parent, list):
parent = parent.split('-')
if not isinstance(conjugate, list):
conjugate = conjugate.split('-')
statistics = {} if statistics is None else statistics
if _DEBUG:
printlog(' Mutation parent', '-'.join(parent))
printlog('Mutation conjugate', '-'.join(conjugate))
# The mutations and their optional parameter, if any
acceptable = fit_enough(best_fitness, statistics)
operations = [
point_mutation, insertion, deletion, inversion, transposition,
conjugation
]
parameters = [acceptable, acceptable, min_len, None, None, None]
child = None
# Repeat until we get a useful mutation
while child is None or child == parent or child == conjugate or '-'.join(
child) in tried:
# Deep copy the parent into child, choose a mutation type, and
# call the appropriate mutation function
child = parent[:]
# I admit, this is a bit tricky! Will generate 5 for the first
# possible choice, 4 for the next, etc. Then use negative indexing
# to choose the right function!
todo = len([i for i in odds if random.randrange(sum(odds)) < i])
child = operations[-todo](child, conjugate, parameters[-todo])
# Rearrange any adjacent modifier codons into a consistent order so we don't end up training
# two effectively identical genomes.
child = regularize(child)
# Quick retry if known to be dead
if child == parent or child == conjugate or '-'.join(child) in tried:
continue
# Check for invalid and nonviable children
if not valid(child) or (viable != None and not viable(child)):
child = None
else:
model, layer_count = build_model(child)
if model is None or layer_count > max_len:
child = None
if _DEBUG:
printlog('New child', '-'.join(child))
return child
def train(org, config, epochs=1):
""" Train an organism for 1 or more epochs
org Organism class instance [genome, fitness, epochs, boolean]
config ModelIO configuration
epochs How many epochs to run
Returns updated organism
"""
if KERNELS is None:
printlog('Fatal Error: configure_environment was not called!')
exit(1)
print('Conserved ', CONSERVED_CODONS)
genome = org.genome
if not isinstance(genome, list):
genome = genome.split('-')
printlog('Training for {} epoch{} : {}'.format(epochs,
's' if epochs > 0 else '',
'-'.join(genome)))
cell = BaseSRCNNModel(org.genome, config)
if cell.model is None:
printlog("Compiling model")
model, _ = build_model(genome,
shape=config.image_shape,
learning_rate=config.learning_rate,
metrics=[cell.evaluation_function])
if model is None:
return Organism([org.genome, 0.0, 0, False])
cell.model = model
else:
printlog("Using loaded model...")
# Now we have a compiled model, execute it - or at least try to, there are still some
# models that may bomb out.
try:
results = cell.fit(run_epochs=epochs)
except KeyboardInterrupt:
raise
except:
printlog('Cannot train: {}'.format(sys.exc_info()[1]))
raise
printlog('Fitness: {}'.format(results))
return Organism([
org.genome, results, org.epoch + epochs, org.improved
and results < org.fitness
])
def on_train_begin(self, logs=None):
if self.verbose:
printlog('Training commences...')
def fit(self, max_epochs=255, run_epochs=0):
""" Train a model.
Uses images in self.config.paths['training'] for Training
Uses images in self.config.paths['validation'] for Validation
"""
"""
printlog('fit')
for key in self.config.config:
if key != 'paths':
printlog(key, self.config.config[key])
"""
samples_per_epoch = self.config.train_images_count()
val_count = self.config.val_images_count()
learning_rate = callbacks.ReduceLROnPlateau(
monitor=self.val_loss_function,
mode='max',
factor=0.9,
min_lr=0.0002,
patience=10,
verbose=self.config.verbose)
# GPU. mode was 'max', but since we want to minimize the PSNR (better = more
# negative) shouldn't it be 'min'?
# Alex: Thats on me, I negated the metric by accident. You wanna max it
model_checkpoint = callbacks.ModelCheckpoint(
self.config.paths['model'],
monitor=self.val_loss_function,
save_best_only=True,
verbose=self.config.verbose,
mode='max',
save_weights_only=False)
# Set up the model state. Can potentially load saved state.
model_state = ModelState(self.config)
# If we have trained previously, set up the model checkpoint so it won't save
# until it finds something better. Otherwise, it would always save the results
# of the first epoch.
if 'best_values' in model_state.state and self.val_loss_function in model_state.state[
'best_values']:
model_checkpoint.best = model_state.state['best_values'][
self.val_loss_function]
if self.config.verbose:
printlog('Best {} found so far: {}'.format(self.val_loss_function,
model_checkpoint.best))
callback_list = [model_checkpoint, learning_rate, model_state]
if self.config.verbose:
printlog('Training model : {}'.format(
self.config.config['model_type']))
# Offset epoch counts if we are resuming training.
initial_epoch = model_state.state['epoch_count']
epochs = max_epochs if run_epochs <= 0 else initial_epoch + run_epochs
# PU: There is an inconsistency when Keras prints that it has saved an improved
# model. It reports that it happened in the previous epoch.
self.model.fit_generator(
self.config.training_data_generator(),
steps_per_epoch=samples_per_epoch // self.config.batch_size,
epochs=epochs,
callbacks=callback_list,
verbose=self.config.bargraph,
validation_data=self.config.validation_data_generator(),
validation_steps=val_count // self.config.batch_size,
initial_epoch=initial_epoch)
if self.config.verbose:
if self.config.bargraph:
print('')
print(' Training results for : {}'.format(self.name))
for key in ['loss', self.loss_function]:
if key in model_state.state['best_values']:
print('{0:>30} : {1:16.10f} @ epoch {2}'.format(
key, model_state.state['best_values'][key],
model_state.state['best_epoch'][key]))
vkey = 'val_' + key
print('{0:>30} : {1:16.10f} @ epoch {2}'.format(
vkey, model_state.state['best_values'][vkey],
model_state.state['best_epoch'][vkey]))
print('')
# PU: Changed to return the best validation results
return model_state.state['best_values']['val_' + self.loss_function]
def on_train_begin(self, logs=None):
if self.verbose:
printlog('Starting Gradient Clipping Value: %f' %
(self.theta / self.learning_rate))
self.optimizer.clipvalue.set_value(self.theta / self.learning_rate)
def configure_environment(env=''):
""" Configure environment globals. env selects a particular
predefined environment - can be a comma-separated list
of environments that are applied in order.
"""
global FILTERS
global KERNELS
global ACTS
global DEPTHS
global MULTIPLIERS
global MERGERS
global MUTABLE_CONVOLUTIONS
global MUTABLE_COMPOSITES
global MUTABLE_MODIFIERS
global PRIMORDIAL_CODONS
global CONSERVED_CODONS
# Defaults
FILTERS = [32, 64]
KERNELS = [3, 5, 7, 9]
ACTS = ['elu']
DEPTHS = [2, 3]
MULTIPLIERS = [1, 2, 4, 8]
MERGERS = {
'add': Add(),
'avg': Average(),
'mult': Multiply(),
'max': Maximum()
}
conserved = []
# Process environmental restrictions - only one for now
for cenv in env.split(','):
if cenv == 'lockio':
conserved += [k for k in {**INPUTS, **OUTPUTS}]
else:
printlog('Warning: Unknown environment {} - ignored!'.format(cenv))
# Set up dictionaries
if ACTS and KERNELS and FILTERS:
MUTABLE_CONVOLUTIONS = {
'conv_f{}_k{}_{}'.format(f, k, a): _bn_conv2d(f,
k,
activation=a,
padding='same')
for a in ACTS for k in KERNELS for f in FILTERS
}
if MERGERS and ACTS and KERNELS and FILTERS:
MUTABLE_COMPOSITES = _make_composites(MERGERS, FILTERS, ACTS, DEPTHS,
KERNELS)
if MULTIPLIERS:
MUTABLE_MODIFIERS = {'mod_r' + str(n): n for n in MULTIPLIERS}
PRIMORDIAL_CODONS = [
k for k in {
**MUTABLE_CONVOLUTIONS,
**MUTABLE_COMPOSITES,
**MUTABLE_MODIFIERS
}
]
CONSERVED_CODONS = conserved
def build_model(genome, shape=(64, 64, 3), learning_rate=0.001, metrics=None):
""" Build and compile a keras model from an expressed sequence of codons.
Returns (model, layer_count) tuple or None if the compile failed in some way
genome list of codon names (if string, will be converted)
shape shape of model input
learning_rate initial learning rate
metrics callbacks
"""
if KERNELS is None:
printlog('Fatal Error: configure_environment was not called!')
exit(1)
# Remove any old layers from global
while _LAYERS:
_LAYERS.pop()
if not isinstance(genome, list):
genome = genome.split('-')
if _DEBUG:
printlog('Compiling', genome)
try:
# Initial layer stacking depth, will be adjusted by modifier codons
depth = 1
# Initial model state, just an input layer
first_layer = Input(shape=shape, dtype='float32')
last_layer = first_layer
_LAYERS.append(first_layer)
# Build the layers of the model.
for i, codon in enumerate(genome):
# If a modifier codon, adjust the depth of the model. We may encounter
# several modifier codons in a row, they are additive. If genome tries
# to modify an output codon, abort with a failure.
if codon in MODIFIERS:
if genome[i + 1] not in OUTPUTS:
depth += ALL_CODONS[codon] - 1
else:
printlog(
'Cannot compile: Modifier codon trying to modify output layer.'
)
return None, 0
else:
# Add the new layer or layer stack and reset the depth
last_layer = ALL_CODONS[codon](last_layer, depth)
depth = 1
# This debug print code assumes Tensorflow is being used.
if _DEBUG:
layer_outputs = {}
for layer in _LAYERS:
lname = layer.name
layer_outputs[lname] = ' + '.join(
[l.name for l in layer.consumers()])
nwidth = max([len(l) for l in layer_outputs])
fstr = 'Layer {:>3d}: {:>' + str(nwidth) + 's} -> {}'
for i, layer in enumerate(_LAYERS):
lname = layer.name
printlog(fstr.format(i, lname, layer_outputs[lname]))
# Create and compile the model
model = Model(first_layer, last_layer)
adam = optimizers.Adam(lr=learning_rate,
clipvalue=(1.0 / .001),
epsilon=0.001)
model.compile(optimizer=adam,
loss='mse',
metrics={} if metrics is None else metrics)
if _DEBUG:
printlog('Compiled model: shape={}, learning rate={}, metrics={}'.
format(shape, learning_rate, metrics))
return (model, len(_LAYERS))
except KeyboardInterrupt:
raise
except:
printlog('Cannot compile: {}'.format(sys.exc_info()[1]))
raise
def extract_tiles(file_path, config, can_disable=False):
""" Helper function that reads in a file, extracts the tiles, and caches them if possible. Handles
size conversion if needed. Note that it cannot handle the quality tile reduction since that
has to be matched between the alpha and beta tiles
"""
global CACHED_TILES
global CACHING
global RESIZE_WARNING
# Cache hit?
if file_path in CACHED_TILES:
return CACHED_TILES[file_path]
img = imread(file_path)
# If we just read in an image that is not the expected size, we need to scale.
# The resolutions we currently are likely to see are 640x480, 720x480 and
# 720x486. In the latter case we chop off 3 rows top and bottom to get 720x480
# before scaling. When we upscale, we do so into the trimmed image area.
shape = np.shape(img)
if shape[0] > config.image_height or shape[1] > config.image_width:
if RESIZE_WARNING:
printlog('Warning: Read image larger than expected {} - downscaling'.format(shape))
printlog('(This warning will not repeat)')
RESIZE_WARNING = False
img = tf.resize(img,
(config.image_height, config.image_width, 3),
order=1,
mode='constant')
elif shape[0] < config.image_height or shape[1] < config.image_width:
# Handle 486 special-case
if shape[0] == 486:
img = img[3:-3, :, :]
shape = np.shape(img)
#trimmed_height = config.expected_height - config.trim_top - config.trim_bottom
#trimmed_width = expected_width - trim_left - trim_right
img = tf.resize(img,
(config.trimmed_height, config.trimmed_width, 3),
order=1,
mode='constant')
if RESIZE_WARNING:
printlog('Warning - Read image smaller than expected {} - upscaled to {}'.format(shape, np.shape(img)))
printlog('(This warning will not repeat)')
RESIZE_WARNING = False
elif config.trim_top + config.trim_bottom + config.trim_left + config.trim_right > 0:
# Input image is expected size, but we have to trim it
img = img[config.trim_top:shape[0] - config.trim_bottom,
config.trim_left:shape[1] - config.trim_right, :]
# Shape may have changed due to all of the munging above
shape = np.shape(img)
# Generate image tile offsets
if len(shape) != 3 or (shape[0] > config.tiles_down * config.base_tile_height) or (shape[1] > config.base_tile_width * config.tiles_across):
printlog('Tesselation Error: file {} has incorrect shape {}'.format(file_path, str(shape)))
return []
# Pad the image - if border_mode is 'constant', the pixels added have
# value (black_level, black_level, black_level). If the mode is 'edge',
# they copy the edge values.
if config.border_mode == 'constant':
img = np.pad(img,
((config.border, config.border), (config.border, config.border), (0, 0)),
mode=config.border_mode, constant_values=config.black_level)
else:
img = np.pad(img,
((config.border, config.border), (config.border, config.border), (0, 0)),
mode=config.border_mode)
offsets = []
# Jittered offsets are shifted half a tile across and down. We need them if
# jitter is set or edge is not set.
if config.jitter or not config.edges:
half_across = config.tile_width // 2
half_down = config.tile_height // 2
jittered_offsets = [(row * config.base_tile_height + half_down, col * config.base_tile_width + half_across)
for row in range(0, config.tiles_down - 1) for col in range(0, config.tiles_across - 1)]
offsets.extend(jittered_offsets)
# Unjittered tile offsets, with optional exclusion of edge tiles. We don't need
# them if edges and jitter are both false
if config.edges or config.jitter:
inset = 0 if config.edges else 1
unjittered_offsets = [(row * config.base_tile_height, col * config.base_tile_width)
for row in range(inset, config.tiles_down-inset) \
for col in range(inset, config.tiles_across-inset)]
offsets.extend(unjittered_offsets)
# Extract tiles from the image
tiles = [img[rpos:rpos + config.tile_height, cpos:cpos + config.tile_width, :]
for (rpos, cpos) in offsets]
# Theano transposition (I hope!)
if config.theano:
tiles = tiles.transpose((0, 3, 1, 2))
# Cache the tiles if possible. We can make sure the cache doesn't turn off by
# setting must_cache. This lets us ensure that pairs of tiles are both cached.
if CACHING:
CACHED_TILES[file_path] = tiles
mem = psutil.virtual_memory()
if can_disable and mem.free < MINFREEMEMORY:
CACHING = False
print('')
print('-----------------------------------------')
print('Cache is full : {} images in cache'.format(len(CACHED_TILES)))
print('Memory status : {}'.format(mem))
print('MINFREEMEMORY : {}'.format(MINFREEMEMORY))
print('-----------------------------------------')
print('')
return tiles
def evolve(config, genepool, image_info):
""" Evolve the genepool """
# Initialize missing values in genepool
if 'population' in genepool:
population = genepool['population']
else:
printlog('Initializing population...')
population = [
["conv_f64_k9_elu-conv_f32_k1_elu-out_k5_elu", 0.0, 0, True],
[
"conv_f64_k9_elu-conv_f32_k1_elu-avg_f32_k135_d3_elu-out_k5_elu",
0.0, 0, True
]
]
# Convert json lists to Organisms
population = [Organism(p) for p in population]
if 'graveyard' in genepool:
graveyard = genepool['graveyard']
graveyard = [Organism(g) for g in graveyard]
else:
printlog('Initializing graveyard...')
graveyard = []
if 'statistics' in genepool:
statistics = genepool['statistics']
else:
printlog('Initializing statistics...')
statistics = {}
poolpath = config.paths['genepool']
# Remind user what we're about to do.
print(' Genepool : {}'.format(config.paths['genepool']))
print(' Environment : {}'.format(config.config['env']))
print(' Tile Width : {}'.format(config.base_tile_width))
print(' Tile Height : {}'.format(config.base_tile_height))
print(' Tile Border : {}'.format(config.border))
print(' Min Population : {}'.format(MIN_POPULATION))
print(' Max Population : {}'.format(MAX_POPULATION))
print(' Epochs to train : {}'.format(config.epochs))
print(' Data root path : {}'.format(config.paths['data']))
print(' Training Images : {}'.format(config.paths['training']))
print(' Validation Images : {}'.format(config.paths['validation']))
print(' Input Image Size : {} x {}'.format(config.image_width,
config.image_height))
print(' Trimming : Top={}, Bottom={}, Left={}, Right={}'.format(
config.trim_top, config.trim_bottom, config.trim_left,
config.trim_right))
print(' Output Image Size : {} x {}'.format(config.trimmed_width,
config.trimmed_height))
print(' Training Set Size : {}'.format(len(image_info[0][0][0])))
print(' Valid. Set Size : {}'.format(len(image_info[1][0][0])))
print(' Black level : {}'.format(config.black_level))
print(' Jitter : {}'.format(config.jitter == 1))
print(' Shuffle : {}'.format(config.shuffle == 1))
print(' Skip : {}'.format(config.skip == 1))
print(' Residual : {}'.format(config.residual == 1))
print(' Quality : {}'.format(config.quality))
checkpoint(poolpath, population, graveyard, statistics, config)
# Set up environmental restrictions.
configure_environment(config.config['env'])
# Repeat until program terminated.
best_fitness = 0
while True:
# While there are some genomes with less than EPOCHS epochs of fitting,
# evolve them 1 epoch and remove the worst performer if it has not shown
# continuous improvement (a sign of a slow evolver). In addition, continue
# to train models past EPOCHS epochs for as long as they show improvement
# in each epoch.
# All sequences that end in a checkpoint are protected by dummy try/except
# blocks, to ensure that a user-break doesn't cause an incorrect state to
# be checkpointed
while population:
# Sanity check for duplicate organisms (could be caused by dumb meatbag
# hand-editing genepool.json)
genomes = [p.genome for p in population]
if len(genomes) != len(set(genomes)):
print('Population contains duplicate genomes! Quitting')
exit(1)
# What organisms need evolution?
todo = [
(i, p) for i, p in enumerate(population)
if p.epoch < EPOCHS or (p.epoch < SLOW_EPOCHS and p.improved)
]
if not todo:
break
# Do the least-trained ones first (in case we got interrupted/restarted)
least_evolved = min([p.epoch for _, p in todo])
todo = [(i, p) for i, p in todo if p.epoch == least_evolved]
epoch_count = max(1, MAX_PER_TRAIN - least_evolved)
# Give them some training
printlog(
'Processing round of {} organisms for {} epoch(s)...'.format(
len(todo), epoch_count))
for i, organism in todo:
config.paths['model'] = os.path.join(config.paths['genebank'],
organism.genome + '.h5')
config.paths['state'] = os.path.join(config.paths['genebank'],
organism.genome + '.json')
config.model_type = organism.genome
config.config['model_type'] = config.model_type
config.config['paths'] = config.paths
config.epochs = 0
config.run_epochs = epoch_count
try:
population[i] = train(organism, config, epochs=epoch_count)
except:
raise
else:
checkpoint(poolpath, population, graveyard, statistics,
config)
# Possibly delete the worst-performer(s), but only during regular evolution
try:
# Sort lowest-fitness to front of list.
population.sort(key=lambda o: o.fitness)
# Remove the worst organism(s), but don't remove if they are continuous
# improvers.
least_evolved = min(EPOCHS, *[p.epoch for p in population])
max_allowed = MAX_POPULATION - least_evolved
if len(population) > max_allowed:
for slot in reversed(range(max_allowed, len(population))):
printlog('Removing {} = {} @ {}'.format(
population[slot].genome, population[slot].fitness,
population[slot].epoch))
graveyard.append(population[slot])
statistics = ligate(statistics,
population[slot].genome,
population[slot].fitness)
del population[slot]
except:
raise
else:
checkpoint(poolpath, population, graveyard, statistics, config)
# Now that training rounds are all done, keep only the best for the next generation
if len(population) > MIN_POPULATION:
try:
# Ye olde survival of the fittest
population.sort(key=lambda o: o.fitness)
# Gather statistics on the genomes that are about to be culled
for organism in population[MIN_POPULATION:]:
printlog('Culling {} = {} @ {}'.format(
organism.genome, organism.fitness, organism.epoch))
statistics = ligate(statistics, organism.genome,
organism.fitness)
graveyard.append(organism)
# Cull the population
population = population[:MIN_POPULATION]
except:
raise
else:
checkpoint(poolpath, population, graveyard, statistics, config)
# Expand the population to the maximum size.
try:
parents, children = [p.genome for p in population], []
tried = [p.genome for p in graveyard] + parents
printlog('Creating new children...')
while len(children) < (MAX_POPULATION - len(parents)):
parent, conjugate = [p for p in random.sample(parents, 2)]
child = '-'.join(
mutate(parent,
conjugate,
tried,
best_fitness=best_fitness,
statistics=statistics))
if child not in parents and child not in children and child not in tried:
children.append(child)
tried.append(child)
else:
printlog('Duplicate genome rejected...')
population.extend([Organism(c) for c in children])
except:
raise
else:
checkpoint(poolpath, population, graveyard, statistics, config)
def kernel_sequence(kernels, number):
""" Generate a random sorted kernel sequence string """
printlog('Kernel sequence :', kernels, number)
return '.'.join(sorted([str(n) for n in random.sample(kernels, number)]))
def point_mutation(child, _, acceptable_fitness):
""" Make a point mutation in a codon. Codon will always be in the format
type_parameter_parameter_... {_activation} and parameter will always
be in format letter-code[digits]. The type is never changed, and we
do special handling for the activation function if it is present.
Currently modifier codons do not have activation functions.
"""
if _DEBUG:
printlog('Point Mutation <', child)
locus = random.randrange(len(child))
original_locus = child[locus]
codons = original_locus.split('_')
basepair = random.randrange(len(codons))
has_depth = any([c[0] == 'd' for c in codons])
# If the chosen codon is highly conserved, do not mess with it.
if original_locus in CONSERVED_CODONS:
if _DEBUG:
printlog('Conserved Codon =', original_locus)
return child
while True:
if basepair == len(codons) - 1 and original_locus in HAS_ACTIVATION:
# choose new activation function
new_codon = random.choice(ACTS)
elif basepair == 0:
# choose new codon type (only if a merge-type codon)
new_codon = random.choice(list(
MERGERS.keys())) if codons[0] in MERGERS else codons[0]
else:
# tweak a codon parameter
base = codons[basepair][0]
param = codons[basepair][1:]
# possible base codes are:
# k kernel size
# d depth of merger codon
# f number of filters
# r replication number of modifier codon
ktype = KERNELS if original_locus in HAS_ACTIVATION else SIMPLE_KERNELS
if base == 'k':
# If the codon has a depth parameter we need a sequence of kernel sizes.
# If we are tweaking a codon with no activation, it's an input/output codon
# so we have a bigger range.
param = kernel_sequence(ktype, len(
param.split('.'))) if has_depth else random.choice(ktype)
elif base == 'd':
# If we change the depth we have to also change the k parameter
param = random.choice(DEPTHS)
codons = [
c if c[0] != 'k' else 'k' + kernel_sequence(ktype, param)
for c in codons
]
elif base == 'f':
param = random.choice(FILTERS)
elif base == 'r':
param = random.choice(MULTIPLIERS)
else:
printlog('Unknown parameter base {} in {}'.format(
base, original_locus))
param = 'XXX'
new_codon = base + str(param)
if acceptable_fitness(new_codon):
break
codons[basepair] = new_codon
child[locus] = '_'.join(codons)
if _DEBUG:
printlog('Point Mutation >', child)
return child
def setup(options):
"""Set up configuration """
# Set up our initial state. Choosing to use a wide border because I was
# seeing tile edge effects.
errors = False
genepool = {}
options.setdefault('border', 10)
options.setdefault('env', '')
options['paths'].setdefault('genepool',
os.path.join('Data', 'genepool.json'))
poolpath = options['paths']['genepool']
if os.path.exists(poolpath):
if os.path.isfile(poolpath):
printlog('Loading existing genepool')
try:
with open(poolpath, 'r') as jsonfile:
genepool = json.load(jsonfile)
# Change 'io' key to 'config' (backwards-compatibility)
if 'io' in genepool:
genepool['config'] = genepool['io']
del genepool['io']
except json.decoder.JSONDecodeError:
printlog(
'Could not parse json. Did you edit "population" and forget to delete the trailing comma?'
)
errors = True
else:
errors = oops(errors, True,
'Genepool path is not a reference to a file ({})',
poolpath)
else:
errors = oops(errors,
not os.access(os.path.dirname(poolpath), os.W_OK),
'Genepool folder is not writeable ({})', poolpath)
terminate(errors, False)
# Genepool settings override config, so we need to update them
for setting in genepool['config']:
if setting not in options or options[setting] != genepool['config'][
setting]:
options[setting] = genepool['config'][setting]
# Reload config with possibly changed settings
config = ModelIO(options)
# Validation and error checking
import Modules.frameops as frameops
image_paths = ['training', 'validation']
sub_folders = ['Alpha', 'Beta']
image_info = [[[], []], [[], []]]
for fcnt, fpath in enumerate(image_paths):
for scnt, _ in enumerate(sub_folders):
image_info[fcnt][scnt] = frameops.image_files(
os.path.join(config.paths[fpath], sub_folders[scnt]), True)
for fcnt in [0, 1]:
for scnt in [0, 1]:
errors = oops(errors, image_info[fcnt][scnt] is None,
'{} images folder does not exist',
image_paths[fcnt] + '/' + sub_folders[scnt])
terminate(errors, False)
for fcnt in [0, 1]:
for scnt in [0, 1]:
errors = oops(errors,
len(image_info[fcnt][scnt]) == 0,
'{} images folder does not contain any images',
image_paths[fcnt] + '/' + sub_folders[scnt])
errors = oops(
errors,
len(image_info[fcnt][scnt]) > 1,
'{} images folder contains more than one type of image',
image_paths[fcnt] + '/' + sub_folders[scnt])
terminate(errors, False)
for fcnt in [0, 1]:
errors = oops(
errors,
len(image_info[fcnt][0][0]) != len(image_info[fcnt][1][0]),
'{} images folders have different numbers of images',
image_paths[fcnt])
terminate(errors, False)
for fcnt in [0, 1]:
for path1, path2 in zip(image_info[fcnt][0][0],
image_info[fcnt][1][0]):
path1, path2 = os.path.basename(path1), os.path.basename(path2)
errors = oops(
errors, path1 != path2,
'{} images folders do not have identical image filenames ({} vs {})',
(image_paths[fcnt], path1, path2))
terminate(errors, False)
# test_files = [[image_info[f][g][0][0] for g in [0, 1]] for f in [0, 1]]
test_images = [[frameops.imread(image_info[f][g][0][0]) for g in [0, 1]]
for f in [0, 1]]
# What kind of file is it? Do I win an award for the most brackets?
# img_suffix = os.path.splitext(image_info[0][0][0][0])[1][1:]
# Check that the Beta tiles are the same size.
size1, size2 = np.shape(test_images[0][1]), np.shape(test_images[1][1])
errors = oops(
errors, size1 != size2,
'Beta training and evaluation images do not have identical size ({} vs {})',
(size1, size2))
# Warn if we do have some differences between Alpha and Beta sizes
for fcnt in [0, 1]:
size1, size2 = np.shape(test_images[fcnt][0]), np.shape(
test_images[fcnt][1])
if size1 != size2:
printlog(
'Warning: {} Alpha and Beta images are not the same size. Will attempt to scale Alpha images.'
.format(image_paths[fcnt].title()))
terminate(errors, False)
# Only check the size of the Beta output for proper configuration, since Alpha tiles will
# be scaled as needed.
errors = oops(errors,
len(size2) != 3 or size2[2] != 3,
'Images have improper shape ({0})', str(size2))
terminate(errors, False)
image_width, image_height = size2[1], size2[0]
trimmed_width = image_width - (config.trim_left + config.trim_right)
trimmed_height = image_height - (config.trim_top + config.trim_bottom)
errors = oops(errors, trimmed_width <= 0,
'Trimmed images have invalid width ({} - ({} + {}) <= 0)',
(size1[0], config.trim_left, config.trim_right))
errors = oops(errors, trimmed_width <= 0,
'Trimmed images have invalid height ({} - ({} + {}) <= 0)',
(size1[1], config.trim_top, config.trim_bottom))
terminate(errors, False)
errors = oops(
errors, (trimmed_width % config.base_tile_width) != 0,
'Trimmed images do not evenly tile horizontally ({} % {} != 0)',
(trimmed_width, config.tile_width))
errors = oops(
errors, (trimmed_height % config.base_tile_height) != 0,
'Trimmed images do not evenly tile vertically ({} % {} != 0)',
(trimmed_height, config.tile_height))
terminate(errors, False)
# Attempt to automatically figure out the border color black level, by finding the minimum pixel value in one of our
# sample images. This will definitely work if we are processing 1440x1080 4:3 embedded in 1920x1080 16:19 images.
# Write back any change into config.
if config.black_level < 0:
config.black_level = np.min(test_images[0][0])
config.config['black_level'] = config.black_level
return (config, genepool, image_info)
def predict(config, image_info):
""" Run predictions using the configuration and list of files """
# Create model. Since the model file contains the complete model info, not just the
# weights, we can instantiate it using the base class. So no matter what changes we
# make to the definition of the models in models.py, old model files will still
# work.
sr_model = models.BaseSRCNNModel(name=config.model_type, config=config)
# Need to use unjittered tiles_per_imag
tiles_per_img = config.tiles_across * config.tiles_down
# Process the images
if config.config['test']:
image_info = [image_info[0], image_info[len(image_info) // 2], image_info[-1]] # just do a couple of images
# There is no point in caching tiles since we never revisit them.
frameops.reset_cache(enabled=False)
for img_path in image_info:
printlog('Predicting', os.path.basename(img_path))
# Generate the tiles for the image. Note that tiles is a generator
tiles = frameops.tesselate(img_path, config)
# Create a batch with all the tiles
tile_batch = np.empty((tiles_per_img, ) + config.image_shape)
for idx, tile in enumerate(tiles):
tile_batch[idx] = tile
"""
if DEBUG:
fname = os.path.basename(img_path)
for i in range(0, min(30, tiles_per_img)):
fpath = os.path.join('Temp', 'PNG', fname[:-4] + '-' + str(i) + '-IN.png')
frameops.imsave(fpath, tile_batch[i])
input_image = frameops.grout(tile_batch, config)
fpath = os.path.join('Temp', 'PNG', os.path.basename(img_path)[:-4] + '-IN.png')
frameops.imsave(fpath, input_image)
"""
# Predict the new tiles in relatively small chunks so the GPU doesn't get clogged
predicted_tiles = sr_model.model.predict(tile_batch, min(config.tiles_across, config.tiles_down))
# GPU : Just using this to debug, uncomment to print section to see results for yourself
# debugging: if residual, then the np.mean of tile_batch should be a
# near zero numbers. Testing supports a mean around 0.0003
# Without residual, mean is usually higher
# print('Debug: Residual {} Mean {}'.format(io.residual==1, np.mean(predicted_tiles)))
if config.residual:
predicted_tiles += tile_batch
# Merge the tiles back into a single image
predicted_image = frameops.grout(predicted_tiles, config)
# Save the image
basename = os.path.basename(img_path)
fname, ext = os.path.splitext(basename)
ext = '.png' if config.config['png'] else ext
basename = fname + ext
fpath = os.path.join(config.paths['predict'], config.beta, basename)
frameops.imsave(fpath, predicted_image)
if config.config['diff']:
difference_tiles = np.absolute(predicted_tiles - tile_batch)
difference_image = frameops.grout(difference_tiles, config)
basename = fname + '-diff' + ext
fpath = os.path.join(config.paths['predict'], config.beta, basename)
frameops.imsave(fpath, difference_image)
# Also generate normalized difference image (easier to see)
axes = (1, 2) if config.theano else (0, 1)
maxdiff = np.amax(difference_image)
maxchan = np.amax(difference_image, axes) * 100.0
avgchan = np.average(difference_image, axes) * 100.0
difference_image /= maxdiff
basename = fname + '-ndiff' + ext
fpath = os.path.join(config.paths['predict'], config.beta, basename)
frameops.imsave(fpath, difference_image)
printlog(' Max channel color error: {:6.2f}%, {:6.2f}%, {:6.2f}%'.format(*maxchan))
printlog(' Avg channel color error: {:6.2f}%, {:6.2f}%, {:6.2f}%'.format(*avgchan))
"""
# Debug code to confirm what we are doing
if DEBUG:
fname = os.path.basename(img_path)
for i in range(0, min(30, tiles_per_img)):
fpath = os.path.join('Temp', 'PNG', fname[0:-4] + '-' + str(i) + '-OUT.png')
frameops.imsave(fpath, predicted_tiles[i])
fpath = os.path.join('Temp', 'PNG', os.path.basename(img_path)[:-4] + '-OUT.png')
frameops.imsave(fpath, predicted_image)
"""
printlog('Predictions completed...')
