def run(self, ngen=10, startfile=None, checkpoint=None):
"""Run the genetic algorithm, updating the population over ngen number of generations.
Keywork arguments:
ngen -- number of generations to run the genetic algorithm.
startfile -- File containing existing population (default None)
checkpoint -- File containing existing checkpoint (default None)
Output:
population -- Resulting population after ngen generations.
"""
if startfile:
population = self.readpop(startfile)
else:
population = self.newpopulation()
if checkpoint:
self.writepop(population, filename=f"0_{checkpoint}")
for cur_g in range(1, ngen + 1):
print(f"generation {cur_g} of population size {len(population)}")
population = self.nextgen(population)
seg = Segmentors.algoFromParams(self.hof[0])
mask = seg.evaluate(self.img)
fitness = Segmentors.FitnessFunction(self.mask, mask)
print(f"#BEST - {fitness[0]} - {self.hof[0]}")
if checkpoint:
print(f"Writing Checkpoint file - {checkpoint}")
self.writepop(population, filename=f"{checkpoint}_{cur_g}")
for cur_p in range(len(population)):
logging.getLogger().info(population[cur_p])
return population
def run(self, ngen=10, population=None,startfile=None, checkpoint=None, cp_freq=1):
"""Run the genetic algorithm, updating the population over ngen number of generations.
Keywork arguments:
ngen -- number of generations to run the genetic algorithm.
startfile -- File containing existing population (default None)
checkpoint -- File containing existing checkpoint (default None)
Output:
population -- Resulting population after ngen generations.
"""
if startfile:
try:
print(f"Reading in {startfile}")
population = self.readpop(startfile)
except FileNotFoundError:
print("WARNING: Start file not found")
except:
raise
if not population:
print(f"Inicializing new randome population")
population = self.newpopulation()
if checkpoint:
self.writepop(population, filename=f"{checkpoint}")
for cur_g in range(0, ngen+1):
print(f"generation {cur_g} of population size {len(population)}")
_, population = self.popfitness(population)
bestsofar = self.hof[0]
seg = Segmentors.algoFromParams(bestsofar)
mask = seg.evaluate(self.img)
fitness = Segmentors.FitnessFunction(mask, self.mask)
print(f"#BEST - {fitness} - {bestsofar}")
if checkpoint and cur_g%cp_freq == 0:
print(f"Writing Checkpoint file - {checkpoint}")
copyfile(f"{checkpoint}", f"{checkpoint}.prev")
self.writepop(population, filename=f"{checkpoint}")
for cur_p in range(len(population)):
logging.getLogger().info(population[cur_p])
if cur_g < ngen+1:
population = self.mutate(population)
if checkpoint:
print(f"Writing Checkpoint file - {checkpoint}")
copyfile(f"{checkpoint}", f"{checkpoint}.prev")
self.writepop(population, filename=f"{checkpoint}")
for cur_p in range(len(population)):
logging.getLogger().info(population[cur_p])
return population
def test_countsets():
"""Unit test for countsets function. Checks output is as
expected for a variety of extreme cases."""
assert Segmentors.countsets({0.0: {0.0: 364}, 1.0: {1.0: 12}, 2.0: {1.0: 24}}) ==\
(0, 2, {0.0: 0.0, 1.0: 1.0, 2.0: 1.0})
assert Segmentors.countsets({0.0: {0.0: 358}, 1.0: {0.0: 6, 1.0: 12}, 2.0: {1.0: 24}}) ==\
(6, 2, {0.0: 0.0, 1.0: 1.0, 2.0: 1.0})
assert Segmentors.countsets({0.0: {0.0: 355}, 3.0: {0.0: 4, 1.0: 2}, 2.0: {1.0: 24},\
1.0: {0.0: 5, 1.0: 10}}) == (7, 2, {0.0: 0.0, 3.0: 0.0, 2.0: 1.0, 1.0: 1.0})
assert Segmentors.countsets({0.0: {0.0: 364, 1.0: 36}}) ==\
(36, 1, {0.0: 0.0})
assert Segmentors.countsets({0.0: {0.0: 76}, 1.0: {0.0: 288, 1.0: 36}}) ==\
(36, 1, {0.0: 0.0, 1.0: 0.0})
def monitor(self):
html_path = os.path.join(os.getcwd(), "web_pages", "monitor.html")
if self.best_fit != -1:
# If there is a segmentor then display the images segmentation
img = imageio.imread(self.rgb_filename)
gmask = imageio.imread(self.label_filename)
print(self.best_ind["params"])
self.best_ind["params"]
seg = Segmentors.algoFromParams(self.best_ind["params"])
mask = seg.evaluate(img)
static_dir = os.path.join(os.getcwd(), "public")
imageio.imwrite(os.path.join(static_dir, "mask.jpg"), mask)
code = GeneticSearch.print_best_algorithm_code(self.best_ind["params"])
# Calculate progress bar precentage
percentage = (1 - self.best_ind["fitness"]) * 100
rounded_fitness = float("{0:.2f}".format(self.best_ind["fitness"]))
data = ["", code, self.best_ind["params"], rounded_fitness, percentage]
else:
data = ['style="display:none;"', "", "", "",""]
return fill_html_template(html_path, data)
def test_runAlgo():
"""Unit test for runAlgo function.
Checks to see if the output is what it's supposed to be in this case."""
individual = ['FB', 0, 0, 984, 0.09, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
assert Segmentors.runAlgo(TEST_IM_COLOR, TEST_IM_COLOR[:, :, 0],
individual) == [sys.maxsize]
def periodic_update(n_intevals, src):
if update_best_individual():
img = "Chameleon.jpg"
segmenter = Segmentors.algoFromParams((results[0].get())["params"])
return "/static/" + tasks.evaluate_segmentation.delay(segmenter,
img).get()
return src
def test_Felzenszwalb():
"""Unit test for Felzenszwalb method. Checks if evaluate function output\
is the same as manually running the skimage function."""
fb1 = Segmentors.Felzenszwalb()
assert fb1.evaluate(TEST_IM_COLOR).all() == segmentation.felzenszwalb(\
TEST_IM_COLOR, 984, 0.09, 92, multichannel=True).all()
assert fb1.evaluate(TEST_IM_GRAY).all() == segmentation.felzenszwalb(\
TEST_IM_GRAY, 984, 0.09, 92, multichannel=False).all()
def test_QuickShift():
"""Unit test for QuickShift method. Checks if evaluate function output\
is the same as manually running the skimage function."""
qs1 = Segmentors.QuickShift()
assert qs1.evaluate(TEST_IM_COLOR).all() == segmentation.quickshift(\
TEST_IM_COLOR, ratio=2, kernel_size=5, max_dist=60, sigma=5, random_seed=1).all()
assert qs1.evaluate(TEST_IM_GRAY).all() == segmentation.quickshift(color.gray2rgb(\
TEST_IM_GRAY), ratio=2, kernel_size=5, max_dist=60, sigma=5, random_seed=1).all()
def test_run_algo():
"""Unit test for runAlgo function.
Checks to see if the output is what it's supposed to be in this case."""
individual = Segmentors.segmentor()
data = pipedata()
data.img = TEST_IM_COLOR
data.gmask = TEST_IM_COLOR[:, :, 0]
individual.runAlgo(data)
def func(img=img, mask=gmask, **kwargs):
"""Find mask and fitness for current algorithm. Show masked image."""
print(seg.params["algorithm"])
for k in kwargs:
seg.params[k] = kwargs[k]
mask = seg.evaluate(img)
fit = Segmentors.FitnessFunction(mask, gmask)
fig = showtwo(img, mask)
def continuous_search(input_file,
input_mask,
startfile='',
checkpoint='checkpoint.txt',
best_mask_file="temp_mask.png",
pop_size=10):
img = imageio.imread(input_file)
gmask = imageio.imread(input_mask)
fid_out = open(f"{input_file}.txt", "w+")
#TODO: Read this file in and set population first
#Run the search
my_evolver = GeneticSearch.Evolver(img, gmask, pop_size=pop_size)
best_fitness = 2.0
iteration = 0
while (best_fitness > 0.0):
print(f"running {iteration} iteration")
if (startfile):
population = my_evolver.run(ngen=1, startfile=None)
startfile = None
else:
population = my_evolver.run(ngen=1)
#Get the best algorithm so far
params = my_evolver.hof[0]
#Generate mask of best so far.
seg = Segmentors.algoFromParams(params)
mask = seg.evaluate(img)
fitness = Segmentors.FitnessFunction(mask, gmask)[0]
if (fitness < best_fitness):
best_fitness = fitness
print(
f"\n\n\n\nIteration {iteration} Finess Improved to {fitness}")
my_evolver.writepop(population, filename="checkpoint.pop")
#imageio.imwrite(best_mask_file,mask);
fid_out.write(f"[{iteration}, {fitness}, {params}]\n")
fid_out.flush()
###TODO Output [fitness, seg]
iteration += 1
def conduct_genetic_search(img, gmask, num_gen, pop_size):
"""
Note: this task could be sped up by
rewriting it to send the evaluation and fitness function
calls to other workers as tasks.
"""
# Only needed on some images
# Convert the RGB 3-channel image into a 1-channel image
# gmask = (np.sum(gmask, axis=2) > 0)
download_and_store_image(img)
download_and_store_image(gmask)
img = imageio.imread(get_path(img))
gmask = imageio.imread(get_path(gmask))
my_evolver = GeneticSearch.Evolver(img, gmask, pop_size=pop_size)
# Conduct the genetic search
population = None
for _ in range(num_gen):
# if population is uninitialized
if population is None:
population = my_evolver.run(ngen=1)
else:
# Simulate a generation and store population in population variable
population = my_evolver.run(ngen=1, population=population)
# Take the best segmentor from the hof and use it to segment the rgb image
seg = Segmentors.algoFromParams(my_evolver.hof[0])
mask = seg.evaluate(img)
# Calculate and print the fitness value of the segmentor
fitness = Segmentors.FitnessFunction(mask, gmask)[0]
params = my_evolver.hof[0]
# Combine data into a single object
data = {}
data["fitness"] = fitness
data["params"] = params
return data
def test_Slic():
"""Unit test for Slic method. Checks if evaluate function output\
is the same as manually running the skimage function."""
sc1 = Segmentors.Slic()
assert sc1.evaluate(TEST_IM_COLOR).all() == segmentation.slic(\
TEST_IM_COLOR, n_segments=5, compactness=5, max_iter=3, \
sigma=5, convert2lab=True, multichannel=True).all()
assert sc1.evaluate(TEST_IM_GRAY).all() == segmentation.slic(\
TEST_IM_GRAY, n_segments=5, compactness=5, max_iter=3, \
sigma=5, convert2lab=True, multichannel=False).all()
def test_Chan_Vese():
"""Unit test for Chan_Vese method. Checks if evaluate function output\
is the same as manually running the skimage function."""
cv1 = Segmentors.Chan_Vese()
assert cv1.evaluate(TEST_IM_COLOR).all() == segmentation.chan_vese(\
color.rgb2gray(TEST_IM_COLOR), mu=2.0, lambda1=10, \
lambda2=20, tol=0.001, max_iter=10, dt=0.10).all()
assert cv1.evaluate(TEST_IM_GRAY).all() == segmentation.chan_vese(\
TEST_IM_GRAY, mu=2.0, lambda1=10, \
lambda2=20, tol=0.001, max_iter=10, dt=0.10).all()
def test_Morphological_Chan_Vese():
"""Unit test for Morphological_Chan_Vese method. Checks if evaluate function output\
is the same as manually running the skimage function."""
mcv1 = Segmentors.Morphological_Chan_Vese()
assert mcv1.evaluate(TEST_IM_COLOR).all() == segmentation.morphological_chan_vese(\
color.rgb2gray(TEST_IM_COLOR), iterations=10, init_level_set="checkerboard", \
smoothing=10, lambda1=10, lambda2=20).all()
assert mcv1.evaluate(TEST_IM_GRAY).all() == segmentation.morphological_chan_vese(\
TEST_IM_GRAY, iterations=10, init_level_set="checkerboard", \
smoothing=10, lambda1=10, lambda2=20).all()
def test_MorphGeodesicActiveContour():
"""Unit test for MorphGeodesicActiveContour method. Checks if evaluate function output\
is the same as manually running the skimage function."""
ac1 = Segmentors.MorphGeodesicActiveContour()
assert ac1.evaluate(TEST_IM_COLOR).all() == segmentation.morphological_geodesic_active_contour(\
segmentation.inverse_gaussian_gradient(color.rgb2gray(TEST_IM_COLOR), 0.2, 0.3),\
iterations=10, init_level_set='checkerboard', smoothing=5, threshold='auto',\
balloon=10).all()
assert ac1.evaluate(TEST_IM_GRAY).all() == segmentation.morphological_geodesic_active_contour(\
segmentation.inverse_gaussian_gradient(TEST_IM_GRAY, 0.2, 0.3), iterations=10,\
init_level_set='checkerboard', smoothing=5, threshold='auto', balloon=10).all()
def segmentwidget(img, gmask, params=None, alg=None):
"""Generate GUI. Produce slider for each parameter for the current segmentor.
Show both options for the masked image.
Keyword arguments:
img -- original image
gmask -- ground truth segmentation mask for the image
params -- list of parameter options
alg -- algorithm to search parameters over
"""
if params is None and alg is None:
alg = 'FB'
params = [alg, 0, 0.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\
(1, 1), 0, 'checkerboard', 'checkerboard', 0, 0, 0, 0, 0, 0]
elif params is None and alg is not None:
params = [alg, 0, 0.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\
(1, 1), 0, 'checkerboard', 'checkerboard', 0, 0, 0, 0, 0, 0]
elif params is not None and alg is not None:
params[0] = alg
seg = Segmentors.algoFromParams(params)
widg = dict()
widglist = []
for ppp in seg.paramindexes:
thislist = eval(seg.params.ranges[ppp])
thiswidg = widgets.SelectionSlider(options=tuple(thislist),
disabled=False,
description=ppp,
value=seg.params[ppp],
continuous_update=False,
orientation='horizontal',
readout=True)
widglist.append(thiswidg)
widg[ppp] = thiswidg
def func(img=img, mask=gmask, **kwargs):
"""Find mask and fitness for current algorithm. Show masked image."""
print(seg.params["algorithm"])
for k in kwargs:
seg.params[k] = kwargs[k]
mask = seg.evaluate(img)
fit = Segmentors.FitnessFunction(mask, gmask)
fig = showtwo(img, mask)
plt.title('Fitness Value: ' + str(fit[0]))
layout = widgets.Layout(grid_template_columns='1fr 1fr 1fr')
u_i = widgets.GridBox(widglist, layout=layout)
out = widgets.interactive_output(func, widg)
display(u_i, out)
return seg.params
def test_mutate():
"""Unit test for mutate function. Checks output type and checks test individual
to see if mutation took place successfully."""
copy_child = ['FB', 0, 0, 984, 0.09, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
all_vals = []
params = Segmentors.parameters()
for key in params.pkeys:
all_vals.append(eval(params.ranges[key]))
assert isinstance(GeneticSearch.mutate(copy_child, all_vals, 0.5, True), list)
assert GeneticSearch.mutate(copy_child, all_vals, 0.5, True) ==\
['FB', 1390, 0.173, 984, 0.09, 9927, 587, 0, 0.55, 0, 0, 0, 0, 1000, 0,\
(1, 2), 0, 'disk', 'checkerboard', 9, 2907, -47, (0.0, 0.0, 0.0), 0, 0]
def makeToolbox(pop_size):
"""Make a genetic algorithm toolbox using DEAP. The toolbox uses premade functions
for crossover, mutation, evaluation and fitness.
Keyword arguments:
pop_size -- The size of our population, or how many individuals we have
"""
# Minimizing fitness function
creator.create("FitnessMin", base.Fitness, weights=(-0.000001, ))
creator.create("Individual", list, fitness=creator.FitnessMin)
# The functions that the GA knows
toolbox = base.Toolbox()
# Genetic functions
toolbox.register("mate", skimageCrossRandom) # crossover
#toolbox.register("mutate", mutate) # Mutation
toolbox.register("mutate", Segmentors.mutateAlgo) # Mutation
toolbox.register("evaluate", Segmentors.runAlgo) # Fitness
toolbox.register("select", tools.selTournament, tournsize=5) # Selection
toolbox.register("map", futures.map) # So that we can use scoop
# DO: May want to later do a different selection process
# We choose the parameters, for the most part, random
params = Segmentors.parameters()
for key in params.pkeys:
toolbox.register(key, random.choice, eval(params.ranges[key]))
func_seq = []
for key in params.pkeys:
func_seq.append(getattr(toolbox, key))
# Here we populate our individual with all of the parameters
toolbox.register("individual",
tools.initCycle,
creator.Individual,
func_seq,
n=1)
# And we make our population
toolbox.register("population",
tools.initRepeat,
list,
toolbox.individual,
n=pop_size)
return toolbox
def test_countMatches():
"""Unit test for countMatches function. Checks output is as
expected for a variety of extreme cases."""
# create test image
ground_truth = np.zeros((20, 20))
ground_truth[4:10, 4:10] = 1
inferred = np.zeros((20, 20))
inferred[4:10, 4:6] = 1
inferred[4:10, 6:10] = 2
assert Segmentors.countMatches(inferred, ground_truth) ==\
({0.0: {0.0: 364}, 1.0: {1.0: 12}, 2.0: {1.0: 24}}, 3, 2)
inferred = np.zeros((20, 20))
inferred[4:10, 3:6] = 1
inferred[4:10, 6:10] = 2
assert Segmentors.countMatches(inferred, ground_truth) ==\
({0.0: {0.0: 358}, 1.0: {0.0: 6, 1.0: 12}, 2.0: {1.0: 24}}, 3, 2)
inferred = np.zeros((20, 20))
inferred[4:10, 3:6] = 1
inferred[4:10, 6:10] = 2
inferred[3:5, 3:6] = 3
assert Segmentors.countMatches(inferred, ground_truth) ==\
({0.0: {0.0: 355}, 3.0: {0.0: 4, 1.0: 2}, 2.0: {1.0: 24}, 1.0: {0.0: 5, 1.0: 10}}, 4, 2)
inferred = np.zeros((20, 20))
assert Segmentors.countMatches(inferred, ground_truth) == ({
0.0: {
0.0: 364,
1.0: 36
}
}, 1, 2)
inferred = np.zeros((20, 20))
inferred[1:19, 1:19] = 1
assert Segmentors.countMatches(inferred, ground_truth) ==\
({0.0: {0.0: 76}, 1.0: {0.0: 288, 1.0: 36}}, 2, 2)
def test_print_best_algorithm_code():
"""Unit test for print_best_algorithm_code function.
Checks function output matches method contents it's printing."""
individual = ['FB', 0, 0, 984, 0.09, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
print_statement = "multichannel = False\n\
if len(img.shape) > 2:\n\
multichannel = True\n\
output = skimage.segmentation.felzenszwalb(\n\
img,\n\
984,\n\
0.09,\n\
92,\n\
multichannel=multichannel,\n\
)\n"
assert Segmentors.print_best_algorithm_code(individual) == print_statement
def print_best_algorithm_code(individual):
"""Print usable code to run segmentation algorithm based on an
individual's genetic representation vector."""
ind_algo = Segmentors.algoFromParams(individual)
original_function = inspect.getsource(ind_algo.evaluate)
function_contents = original_function[original_function.find(' '):\
original_function.find('return')]
while function_contents.find('self.params') != -1:
function_contents = function_contents.replace(
function_contents[function_contents.find('self.params'):function_contents.find(']')+1],\
str(ind_algo.params[function_contents[function_contents.find('self.params') + 13:\
function_contents.find(']')-1]]))
function_contents = function_contents.replace(' ', '')
function_contents = function_contents[function_contents.find('\n\"\"\"') +
5:]
print(function_contents)
return function_contents
def print_best_algorithm_code(individual):
"""Print usable code to run segmentation algorithm based on an
individual's genetic representation vector."""
ind_algo = Segmentors.algoFromParams(individual)
original_function = inspect.getsource(ind_algo.evaluate)
# Get the body of the function
function_contents = original_function[original_function.find(' '):\
original_function.find('return')]
while function_contents.find('self.params') != -1:
# Find the index of the 's' at the start of self.params
params_index = function_contents.find('self.params')
# Find the index of the ']' at the end of self.params["<SOME_TEXT>"]
end_bracket_index = function_contents.find(']', params_index) + 1
# Find the first occurance of self.params["<SOME_TEXT>"] and store it
code_to_replace = function_contents[params_index:end_bracket_index]
# These offset will be used to access only the params_key
offset = len('self.params["')
offset2 = len('"]')
# Get the params key
params_key = function_contents[params_index +
offset:end_bracket_index - offset2]
# Use the params_key to access the params_value
param_value = str(ind_algo.params[params_key])
# Replace self.params["<SOME_TEXT>"] with the value of self.params["<SOME_TEXT>"]
function_contents = function_contents.replace(code_to_replace,
param_value)
function_contents = function_contents.replace(' ', '')
function_contents = function_contents[function_contents.find('\n\"\"\"') +
5:]
print(function_contents)
return function_contents
def test_FitnessFunction():
"""Unit test for FitnessFunction function. Checks fitness value is as
expected for a variety of extreme cases."""
# create test image
ground_truth = np.zeros((20, 20))
ground_truth[4:10, 4:10] = 1
inferred = np.zeros((20, 20))
inferred[4:10, 4:6] = 1
inferred[4:10, 6:10] = 2
assert Segmentors.FitnessFunction(inferred, ground_truth) == [
2**np.log(3),
]
inferred = np.zeros((20, 20))
inferred[4:10, 3:6] = 1
inferred[4:10, 6:10] = 2
assert Segmentors.FitnessFunction(inferred, ground_truth) == [
8**np.log(3),
]
inferred = np.zeros((20, 20))
inferred[4:10, 3:6] = 1
inferred[4:10, 6:10] = 2
inferred[3:5, 3:6] = 3
assert Segmentors.FitnessFunction(inferred, ground_truth) == [
9**np.log(4),
]
inferred = np.zeros((20, 20))
assert Segmentors.FitnessFunction(inferred, ground_truth) == [
sys.maxsize,
]
inferred = np.arange(400).reshape(ground_truth.shape)
assert Segmentors.FitnessFunction(inferred, ground_truth) == [
2**np.log(400),
]
inferred = np.zeros((20, 20))
inferred[1:19, 1:19] = 1
assert Segmentors.FitnessFunction(inferred, ground_truth) == [
sys.maxsize,
]
args = parser.parse_args()
print(args)
random.seed(args.seed)
logging.basicConfig(stream=sys.stdout, level=logging.ERROR)
#logging.basicConfig(stream=sys.stdout, level=logging.INFO)
img = imageio.imread(args.image)
gmask = imageio.imread(args.mask)
if len(gmask.shape) > 2:
gmask = color.rgb2gray(gmask)
ee = GeneticSearch.Evolver(img, gmask, pop_size=args.pop)
ee.run(args.generations, checkpoint=args.checkpointfile)
seg = Segmentors.algoFromParams(ee.hof[0])
mask = seg.evaluate(img)
imageio.imwrite(args.outputfolder+"file.jpg", mask)
fitness,_ = Segmentors.FitnessFunction(mask,gmask)
print(f"{fitness} {ee.hof[0]}")
def segmentwidget(img, gmask, params=None, alg=None):
"""Generate GUI. Produce slider for each parameter for the current segmentor.
Show both options for the masked image.
Keyword arguments:
img -- original image
gmask -- ground truth segmentation mask for the image
params -- list of parameter options
alg -- algorithm to search parameters over
"""
if params:
if alg:
params[0] = alg;
seg = Segmentors.algoFromParams(params)
else:
if alg:
algorithm_gen = Segmentors.algorithmspace[alg]
seg = algorithm_gen()
else:
seg = Segmentors.segmentor()
widg = dict()
widglist = []
for ppp, ind in zip(seg.paramindexes, range(len(seg.paramindexes))):
thislist = eval(seg.params.ranges[ppp])
name = ppp
current_value = seg.params[ppp]
if not current_value in thislist:
#TODO: We should find the min distance between current_value and this list and use that instead.
current_value = thislist[0]
thiswidg = widgets.SelectionSlider(options=tuple(thislist),
disabled=False,
description=name,
value=current_value,
continuous_update=False,
orientation='horizontal',
readout=True
)
widglist.append(thiswidg)
widg[ppp] = thiswidg
# algorithms = list(Segmentors.algorithmspace.keys())
# w = widgets.Dropdown(
# options=algorithms,
# value=algorithms[0],
# description='Choose Algorithm:',
# )
def func(img=img, mask=gmask, **kwargs):
"""Find mask and fitness for current algorithm. Show masked image."""
print(seg.params["algorithm"])
for k in kwargs:
seg.params[k] = kwargs[k]
mask = seg.evaluate(img)
fit = Segmentors.FitnessFunction(mask, gmask)
fig = showtwo(img, mask)
# I like the idea of printing the sharepython but it should be below the figures.
#print(seg.sharepython(img))
# plt.title('Fitness Value: ' + str(fit[0]))
layout = widgets.Layout(grid_template_columns='1fr 1fr 1fr')
u_i = widgets.GridBox(widglist, layout=layout)
out = widgets.interactive_output(func, widg)
display(u_i, out)
return seg.params
class Evolver(object):
"""Perform the genetic algorithm by initializing a population and evolving it over a
specified number of generations to find the optimal algorithm and parameters for the problem.
Functions:
newpopulation -- Initialize a new population.
writepop -- Records our population in the file "filename".
readpop -- Reads in existing population from "filename".
popfitness -- Calculates the fitness values for our population.
mutate -- Performs mutation and crossover on population.
nextgen -- Generates the next generation of our population.
run -- Runs the genetic algorithm.
"""
AllVals = []
my_p = Segmentors.parameters()
for key in my_p.pkeys:
AllVals.append(eval(my_p.ranges[key]))
def __init__(self, img, mask, pop_size=10):
"""Set default values for the variables.
Keyword arguments:
img -- The original training image
mask -- The ground truth segmentation mask for the img
pop_size -- Integer value denoting size of our population,
or how many individuals there are (default 10)
"""
# Build Population based on size
self.img = img
self.mask = mask
self.tool = makeToolbox(pop_size)
self.hof = deap.tools.HallOfFame(10)
self.best_avgs = []
self.gen = 0
self.cxpb, self.mutpb, self.flip_prob = 0.9, 0.9, 0.9
def newpopulation(self):
"""Initialize a new population."""
return self.tool.population()
def writepop(self, tpop, filename='test.json'):
"""Record the population in the file "filename".
Keyword arguments:
tpop -- The population to be recorded.
filename -- string denoting file in which to record
the population. (default 'test.json')
"""
logging.getLogger().info(f"Writting population to {filename}")
with open(filename, 'w') as outfile:
json.dump(tpop, outfile)
def readpop(self, filename='test.json'):
"""Read in existing population from "filename"."""
logging.getLogger().info(f"Reading population from {filename}")
self.tool.register("population_read", initPopulation,
list, creator.Individual, filename)
self.tool.register("individual_guess",
initIndividual, creator.Individual)
self.tool.register("population_guess", initPopulation,
list, self.tool.individual_guess, "my_guess.json")
return self.tool.population_read()
def popfitness(self, tpop):
"""Calculate the fitness values for the population, and log general statistics about these
values. Uses hall of fame (hof) to keep track of top 10 individuals.
Keyword arguments:
tpop -- current population
Outputs:
extract_fits -- Fitness values for our population
tpop -- current population
"""
new_image = [self.img for i in range(0, len(tpop))]
new_val = [self.mask for i in range(0, len(tpop))]
fitnesses = map(self.tool.evaluate, new_image, new_val, tpop)
# DO: Dirk is not sure exactly why we need these
for ind, fit in zip(tpop, fitnesses):
ind.fitness.values = fit
extract_fits = [ind.fitness.values[0] for ind in tpop]
self.hof.update(tpop)
#Algo = AlgorithmSpace(AlgoParams)
# Evaluating the new population
leng = len(tpop)
mean = sum(extract_fits) / leng
self.best_avgs.append(mean)
sum1 = sum(i*i for i in extract_fits)
stdev = abs(sum1 / leng - mean ** 2) ** 0.5
logging.getLogger().info(f"Generation: {self.gen}")
logging.getLogger().info(f" Min: {min(extract_fits)}")
logging.getLogger().info(f" Max: {max(extract_fits)}")
logging.getLogger().info(f" Avg: {mean}")
logging.getLogger().info(f" Std: {stdev}")
logging.getLogger().info(f" Size: {leng}")
#logging.info(" Time: ", time.time() - initTime)
logging.getLogger().info(f"Best Fitness: {self.hof[0].fitness.values}")
logging.getLogger().info(f"{self.hof[0]}")
# Did we improve the population?
# past_pop = tpop
# past_min = min(extract_fits)
# past_mean = mean
self.gen += self.gen
return extract_fits, tpop
def mutate(self, tpop):
"""Return new population with mutated individuals. Perform both mutation and crossover.
Keyword arguments:
tpop -- current population
Output:
final -- new population with mutated individuals.
"""
# Calculate next population
#TODO: There is an error here. We need to make sure the best hof is included?
my_sz = len(tpop) #Length of current population
top = min(10,max(1,round(0.1 * my_sz)))
top = min(top, len(self.hof))
var = max(1,round(0.4 * my_sz))
var = min(var, len(self.hof))
ran = my_sz - top - var
print(f"pop[0:{top}:{var}:{ran}]")
print(f"pop[0:{top}:{top+var}:{my_sz}]")
# offspring = self.tool.select(tpop, var)
# offspring = list(map(self.tool.clone, offspring)) # original code
offspring = copy.deepcopy(list(self.hof))
# crossover
for child1, child2 in zip(offspring[::2], offspring[1::2]):
# Do we crossover?
if random.random() < self.cxpb:
self.tool.mate(child1, child2)
# The parents may be okay values so we should keep them
# in the set
del child1.fitness.values
del child2.fitness.values
# mutation
for mutant in offspring:
if random.random() < self.mutpb:
self.tool.mutate(mutant, self.AllVals, self.flip_prob)
del mutant.fitness.values
# new
#population = self.newpopulation()
pop = self.tool.population()
final = pop[0:ran]
print(f"pop size should be {len(final)}")
final += self.hof[0:top]
print(f"pop size should be {len(final)}")
final += offspring[0:var]
print(f"pop size should be {len(final)}")
print(f"pop[0:{top}:{var}:{ran}]")
print(f"pop size should be {len(final)}")
# Replacing the old population
return final
def nextgen(self, tpop):
"""Generate the next generation of the population.
Keyword arguments:
tpop -- current population
"""
_, tpop = self.popfitness(tpop)
return self.mutate(tpop)
def run(self, ngen=10, population=None,startfile=None, checkpoint=None, cp_freq=1):
"""Run the genetic algorithm, updating the population over ngen number of generations.
Keywork arguments:
ngen -- number of generations to run the genetic algorithm.
startfile -- File containing existing population (default None)
checkpoint -- File containing existing checkpoint (default None)
Output:
population -- Resulting population after ngen generations.
"""
if startfile:
try:
print(f"Reading in {startfile}")
population = self.readpop(startfile)
except FileNotFoundError:
print("WARNING: Start file not found")
except:
raise
if not population:
print(f"Inicializing new randome population")
population = self.newpopulation()
if checkpoint:
self.writepop(population, filename=f"{checkpoint}")
for cur_g in range(0, ngen+1):
print(f"generation {cur_g} of population size {len(population)}")
_, population = self.popfitness(population)
bestsofar = self.hof[0]
seg = Segmentors.algoFromParams(bestsofar)
mask = seg.evaluate(self.img)
fitness = Segmentors.FitnessFunction(mask, self.mask)
print(f"#BEST - {fitness} - {bestsofar}")
if checkpoint and cur_g%cp_freq == 0:
print(f"Writing Checkpoint file - {checkpoint}")
copyfile(f"{checkpoint}", f"{checkpoint}.prev")
self.writepop(population, filename=f"{checkpoint}")
for cur_p in range(len(population)):
logging.getLogger().info(population[cur_p])
if cur_g < ngen+1:
population = self.mutate(population)
if checkpoint:
print(f"Writing Checkpoint file - {checkpoint}")
copyfile(f"{checkpoint}", f"{checkpoint}.prev")
self.writepop(population, filename=f"{checkpoint}")
for cur_p in range(len(population)):
logging.getLogger().info(population[cur_p])
return population
my_evolver = GeneticSearch.Evolver(img, gmask, pop_size=pop_size)
# Conduct the genetic search
population = None
for i in range(num_gen):
# if population is uninitialized
if population is None:
population = my_evolver.run(ngen=1)
else:
# Simulate a generation and store population in population variable
population = my_evolver.run(ngen=1, population=population)
# Take the best segmentor from the hof and use it to segment the rgb image
seg = Segmentors.algoFromParams(my_evolver.hof[0])
mask = seg.evaluate(img)
# Calculate and print the fitness value of the segmentor
fitness = Segmentors.FitnessFunction(mask, gmask)[0]
params = my_evolver.hof[0]
# Combine data into a single object
data = {}
data["fitness"] = fitness
data["params"] = params
# Convert the data to json format
data = json.dumps(data)
print(i, data, "\n\n")
def run(self, ngen=10, population=None,startfile=None, checkpoint=None, cp_freq=1):
"""Run the genetic algorithm, updating the population over ngen number of generations.
Keywork arguments:
ngen -- number of generations to run the genetic algorithm.
startfile -- File containing existing population (default None)
checkpoint -- File containing existing checkpoint (default None)
Output:
population -- Resulting population after ngen generations.
"""
if startfile:
try:
print(f"Reading in {startfile}")
population = self.readpop(startfile)
except FileNotFoundError:
print("WARNING: Start file not found")
except:
raise
if not population:
print(f"Initializing a new random population")
population = self.newpopulation()
if checkpoint:
self.writepop(population, filename=f"{checkpoint}")
for cur_g in range(0, ngen+1):
print(f"Generation {cur_g} of population size {len(population)}")
histogram = Segmentors.popCounts(population)
print(f"#HIST - {histogram}")
_, population = self.popfitness(population)
bestsofar = self.hof[0]
seg = Segmentors.algoFromParams(bestsofar)
mask = seg.evaluate(self.img)
fitness = Segmentors.FitnessFunction(mask, self.mask)
print(f"#BEST - {fitness} - {bestsofar}")
if checkpoint and cur_g%cp_freq == 0:
print(f"Writing Checkpoint file - {checkpoint}")
copyfile(f"{checkpoint}", f"{checkpoint}.prev")
self.writepop(population, filename=f"{checkpoint}")
for cur_p in range(len(population)):
logging.getLogger().info(population[cur_p])
if cur_g < ngen+1:
if bestsofar.fitness.values[0] >= 0.95:
population = self.newpopulation()
# if the best fitness value is at or above the
# threshold of 0.95, discard the entire current
# population and randomly select a new population
# for the next generation
# note: setting keep_prob = 0 and mutate_prob = 1
# as mutate arguments
# should have same result as self.new_population()
else:
population = self.mutate(population)
# if the best fitness value is below this threshold,
# proceed as normal, mutating the current population
# to get the next generation
if checkpoint:
print(f"Writing Checkpoint file - {checkpoint}")
copyfile(f"{checkpoint}", f"{checkpoint}.prev")
self.writepop(population, filename=f"{checkpoint}")
for cur_p in range(len(population)):
logging.getLogger().info(population[cur_p])
return population
def test_parameter_len():
"""Unit test for parameters function. Checks formatting of parameter."""
param = Segmentors.segmentor().params
assert len(param) > 1
