def continuous_search(input_file,
input_mask,
startfile=None,
checkpoint='checkpoint.txt',
best_mask_file="temp_mask.png",
pop_size=10):
"""Run genetic search continuously.
input_file: the original image
input_mask: the ground truth mask for the image
pop_size: the size of the population
Runs indefinitely unless a perfect value (0.0) is reached.
"""
mydata = base_classes.pipedata()
mydata.img = imageio.imread(input_file)
mydata.gmask = imageio.imread(input_mask)
outfile=f"{input_file}.txt"
#TODO: Read this file in and set population first
workflow.addalgos([colorspace, segmentor, segment_fitness])
wf = workflow()
#Run the search
my_evolver = GeneticSearch.Evolver(workflow, mydata, pop_size=pop_size)
best_fitness=2.0
iteration = 0
while(best_fitness > 0.0):
print(f"running {iteration} iteration")
if(startfile):
population = my_evolver.run(ngen=1, startfile=startfile)
else:
population = my_evolver.run(ngen=1)
#Get the best algorithm so far
params = my_evolver.hof[0]
#Generate mask of best so far.
seg = workflow(paramlist=params)
mydata = seg.pipe(mydata)
fitness = mydata.fitness
if (fitness < best_fitness):
best_fitness = fitness
print(f"\n\n\n\nIteration {iteration} Finess Improved to {fitness}")
my_evolver.writepop(population, filename="checkpoint.pop")
imageio.imwrite(best_mask_file,mydata.mask);
GeneticSearch.write_algo_vector(fpop_file, f"[{iteration}, {fitness}, {params}]\n")
###TODO Output [fitness, seg]
iteration += 1
def test_mutate():
"""Unit test for mutate function. Checks output type and checks test individual
to see if mutation took place successfully."""
copy_child = ['FB', 0, 0, 984, 0.09, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
all_vals = []
params = Segmentors.parameters()
for key in params.pkeys:
all_vals.append(eval(params.ranges[key]))
assert isinstance(GeneticSearch.mutate(copy_child, all_vals, 0.5, True), list)
assert GeneticSearch.mutate(copy_child, all_vals, 0.5, True) ==\
['FB', 1390, 0.173, 984, 0.09, 9927, 587, 0, 0.55, 0, 0, 0, 0, 1000, 0,\
(1, 2), 0, 'disk', 'checkerboard', 9, 2907, -47, (0.0, 0.0, 0.0), 0, 0]
def continuous_search(input_file,
input_mask,
startfile=None,
checkpoint='checkpoint.txt',
best_mask_file="temp_mask.png",
pop_size=10):
"""Run genetic search continuously.
input_file: the original image
input_mask: the ground truth mask for the image
pop_size: the size of the population
Runs indefinitely unless a perfect value (0.0) is reached.
"""
mydata = base_classes.pipedata()
mydata.img = imageio.imread(input_file)
mydata.gmask = imageio.imread(input_mask)
pname = Path(input_file)
outfile=pname.parent.joinpath(f"_{pname.stem}.txt")
mask_file = pname.parent.joinpath(f"{pname.stem}_bestsofar.png")
#TODO: Read this file in and set population first
workflow.addalgos([colorspace, segmentor, segment_fitness])
wf = workflow()
my_evolver = GeneticSearch.Evolver(workflow, mydata, pop_size=pop_size)
population = my_evolver.newpopulation()
best_fitness=2.0
if outfile.exists():
inlist, fitness=read_pop(outfile)
for fit in fitness:
if fit < best_fitness:
best_fitness = fit
previous_pop = my_evolver.copy_pop_list(inlist)
if len(previous_pop) > len(population):
population = previous_pop[:-len(population)]
else:
for index, ind in enumerate(previous_pop):
population[index] = ind
print(f"######### Done importing previous list {best_fitness}")
iteration = 0
while best_fitness > 0.0:
print(f"running {iteration} iteration")
population = my_evolver.run(ngen=1,population=population)
#Get the best algorithm so far
best_so_far = my_evolver.hof[0]
fitness = best_so_far.fitness.values[0]
if (fitness < best_fitness):
best_fitness = fitness
print(f"\n\n\n\nIteration {iteration} Finess Improved to {fitness}")
#Generate mask of best so far.
seg = workflow(paramlist=best_so_far)
mydata = seg.pipe(mydata)
imageio.imwrite(mask_file,mydata.mask);
write_vector(f"{outfile}", f"[{iteration}, {fitness}, {best_so_far}]")
iteration += 1
def monitor(self):
html_path = os.path.join(os.getcwd(), "web_pages", "monitor.html")
if self.best_fit != -1:
# If there is a segmentor then display the images segmentation
img = imageio.imread(self.rgb_filename)
gmask = imageio.imread(self.label_filename)
print(self.best_ind["params"])
self.best_ind["params"]
seg = Segmentors.algoFromParams(self.best_ind["params"])
mask = seg.evaluate(img)
static_dir = os.path.join(os.getcwd(), "public")
imageio.imwrite(os.path.join(static_dir, "mask.jpg"), mask)
code = GeneticSearch.print_best_algorithm_code(self.best_ind["params"])
# Calculate progress bar precentage
percentage = (1 - self.best_ind["fitness"]) * 100
rounded_fitness = float("{0:.2f}".format(self.best_ind["fitness"]))
data = ["", code, self.best_ind["params"], rounded_fitness, percentage]
else:
data = ['style="display:none;"', "", "", "",""]
return fill_html_template(html_path, data)
def test_newpopulation():
"""Unit test for newpopulation function. Checks the type and length of
the new population."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
evolv = GeneticSearch.Evolver(img, mask)
assert isinstance(evolv.tool.population(), list)
assert len(evolv.tool.population()) == 10
def test_mutate():
"""Unit test for mutate function. Checks type and length of the
new population after mutation."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
evolv = GeneticSearch.Evolver(img, mask)
tpop = evolv.mutate(evolv.tool.population())
assert isinstance(tpop, list)
assert len(tpop) == 10
def runsearch(self):
img = imageio.imread(
"Image_data/Coco_2017_unlabeled//rgbd_plant/rgb_00_000_00.png")
gmask = imageio.imread(
"Image_data/Coco_2017_unlabeled/rgbd_new_label/label_00_000_000.png"
)
if len(gmask.shape) > 2:
gmask = color.rgb2gray(gmask)
ee = GeneticSearch.Evolver(img, gmask)
ee.run(self.GENERATIONS)
def test_run():
"""Unit test for run function. Checks the type and length of the final population,
and checks that the population evolved."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
evolv = GeneticSearch.Evolver(img, mask)
start_pop = evolv.tool.population()
final_pop = evolv.run()
assert isinstance(final_pop, list)
assert len(final_pop) == 10
assert final_pop != start_pop
def test_nextgen():
"""Unit test for nextgen function. Checks the type and length of the new population,
and checks that the population is evolving."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
evolv = GeneticSearch.Evolver(img, mask)
pop = evolv.tool.population()
tpop = evolv.mutate(pop)
assert isinstance(tpop, list)
assert len(tpop) == 10
assert tpop != pop
def test_popfitness():
"""Unit test for popfitness function. Checks the type and length of the fitness
values and population."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
evolv = GeneticSearch.Evolver(img, mask)
fits, tpop = evolv.popfitness(evolv.tool.population())
assert isinstance(fits, list)
assert len(fits) == 10
assert isinstance(tpop, list)
assert len(tpop) == 10
def test_skimageCrossRandom():
"""Unit test for skimageCrossRandom function. Checks test individuals to see if crossover
took place successfully."""
np1 = ['FB', 0, 0, 984, 0.09, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
np2 = ['CT', 0, 0, 0, 0, 0, 0, 0, 8, 10, 12, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
new_np1, new_np2 = GeneticSearch.skimageCrossRandom(np1, np2)
assert new_np1 == ['FB', 0, 0, 984, 0.09, 92, 0, 0, 8, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, 'checkerboard', 'checkerboard', 0, 0, 0, 0, 0, 0]
assert new_np2 == ['CT', 0, 0, 0, 0, 0, 0, 0, 0, 10, 12, 0, 0, 0, 0,\
(1, 2), 0, 'checkerboard', 'checkerboard', 0, 0, 0, 0, 0, 0]
def test_twoPointCopy():
"""Unit test for twoPointCopy function. Checks test individuals to see
if copy took place successfully."""
np1 = ['FB', 0, 0, 984, 0.09, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
np2 = ['CT', 0, 0, 0, 0, 0, 0, 0, 8, 10, 12, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
new_np1, new_np2 = GeneticSearch.twoPointCopy(np1, np2, True)
assert new_np1 == ['FB', 0, 0, 984, 0.09, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, 'checkerboard', 'checkerboard', 0, 0, 0, 0, 0, 0]
assert new_np2 == ['CT', 0, 0, 0, 0, 0, 0, 0, 8, 10, 12, 0, 0, 0, 0,\
(1, 2), 0, 'checkerboard', 'checkerboard', 0, 0, 0, 0, 0, 0]
def runsearch(self):
img = imageio.imread(
'Image_data/Coco_2017_unlabeled//rgbd_plant/rgb_00_000_00.png')
gmask = imageio.imread(
'Image_data/Coco_2017_unlabeled/rgbd_new_label/label_00_000_000.png'
)
if len(gmask.shape) > 2:
gmask = color.rgb2gray(gmask)
ee = GeneticSearch.Evolver(img, gmask)
#ee.run(self.GENERATIONS)
population = ee.run(self.GENERATIONS,
startfile="0_checkpoint.json",
checkpoint="2_checkpoint.json")
def test_newpopulation():
"""Unit test for newpopulation function. Checks the type and length of
the new population."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
data = base_classes.pipedata()
data.img = img
data.gmask = mask
data.fitness = 2
evolv = GeneticSearch.Evolver(Segmentors.segmentor, data, pop_size=10)
assert isinstance(evolv.tool.population(), list)
assert len(evolv.tool.population()) == 10
def test_mutate():
"""Unit test for mutate function. Checks type and length of the
new population after mutation."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
data = base_classes.pipedata()
data.img = img
data.gmask = mask
evolv = GeneticSearch.Evolver(Segmentors.segmentor, data, pop_size=10)
tpop = evolv.mutate(evolv.tool.population())
assert isinstance(tpop, list)
assert len(tpop) == 10
def test_nextgen():
"""Unit test for nextgen function. Checks the type and length of the new population,
and checks that the population is evolving."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
data = base_classes.pipedata()
data.img = img
data.gmask = mask
evolv = GeneticSearch.Evolver(Segmentors.segmentor, data, pop_size=10)
pop = evolv.tool.population()
tpop = evolv.mutate(pop)
assert isinstance(tpop, list)
assert len(tpop) == 10
assert tpop != pop
def test_print_best_algorithm_code():
"""Unit test for print_best_algorithm_code function.
Checks function output matches method contents it's printing."""
individual = ['FB', 0, 0, 984, 0.09, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0,\
(1, 2), 0, "checkerboard", "checkerboard", 0, 0, 0, 0, 0, 0]
print_statement = "multichannel = False\n\
if len(img.shape) > 2:\n\
multichannel = True\n\
output = skimage.segmentation.felzenszwalb(\n\
img,\n\
984,\n\
0.09,\n\
92,\n\
multichannel=multichannel,\n\
)\n"
assert GeneticSearch.print_best_algorithm_code(individual) == print_statement
def continuous_search(input_file,
input_mask,
startfile='',
checkpoint='checkpoint.txt',
best_mask_file="temp_mask.png",
pop_size=10):
img = imageio.imread(input_file)
gmask = imageio.imread(input_mask)
fid_out = open(f"{input_file}.txt", "w+")
#TODO: Read this file in and set population first
#Run the search
my_evolver = GeneticSearch.Evolver(img, gmask, pop_size=pop_size)
best_fitness = 2.0
iteration = 0
while (best_fitness > 0.0):
print(f"running {iteration} iteration")
if (startfile):
population = my_evolver.run(ngen=1, startfile=None)
startfile = None
else:
population = my_evolver.run(ngen=1)
#Get the best algorithm so far
params = my_evolver.hof[0]
#Generate mask of best so far.
seg = Segmentors.algoFromParams(params)
mask = seg.evaluate(img)
fitness = Segmentors.FitnessFunction(mask, gmask)[0]
if (fitness < best_fitness):
best_fitness = fitness
print(
f"\n\n\n\nIteration {iteration} Finess Improved to {fitness}")
my_evolver.writepop(population, filename="checkpoint.pop")
#imageio.imwrite(best_mask_file,mask);
fid_out.write(f"[{iteration}, {fitness}, {params}]\n")
fid_out.flush()
###TODO Output [fitness, seg]
iteration += 1
def test_run():
"""Unit test for run function. Checks the type and length of the final population,
and checks that the population evolved."""
img = np.zeros((20, 20, 3))
img[4:10, 4:10, :] = 1
mask = img[:, :, 0]
data = base_classes.pipedata()
data.img = img
data.gmask = mask
data.fitness = 2
evolv = GeneticSearch.Evolver(Segmentors.segmentor, data, pop_size=10)
start_pop = evolv.tool.population()
final_pop = evolv.run()
assert isinstance(final_pop, list)
assert len(final_pop) == 10
assert final_pop != start_pop
def conduct_genetic_search(img, gmask, num_gen, pop_size):
"""
Note: this task could be sped up by
rewriting it to send the evaluation and fitness function
calls to other workers as tasks.
"""
# Only needed on some images
# Convert the RGB 3-channel image into a 1-channel image
# gmask = (np.sum(gmask, axis=2) > 0)
download_and_store_image(img)
download_and_store_image(gmask)
img = imageio.imread(get_path(img))
gmask = imageio.imread(get_path(gmask))
my_evolver = GeneticSearch.Evolver(img, gmask, pop_size=pop_size)
# Conduct the genetic search
population = None
for _ in range(num_gen):
# if population is uninitialized
if population is None:
population = my_evolver.run(ngen=1)
else:
# Simulate a generation and store population in population variable
population = my_evolver.run(ngen=1, population=population)
# Take the best segmentor from the hof and use it to segment the rgb image
seg = Segmentors.algoFromParams(my_evolver.hof[0])
mask = seg.evaluate(img)
# Calculate and print the fitness value of the segmentor
fitness = Segmentors.FitnessFunction(mask, gmask)[0]
params = my_evolver.hof[0]
# Combine data into a single object
data = {}
data["fitness"] = fitness
data["params"] = params
return data
args = parser.parse_args()
print(args)
random.seed(args.seed)
logging.basicConfig(stream=sys.stdout, level=logging.ERROR)
#logging.basicConfig(stream=sys.stdout, level=logging.INFO)
img = imageio.imread(args.image)
gmask = imageio.imread(args.mask)
if len(gmask.shape) > 2:
gmask = color.rgb2gray(gmask)
ee = GeneticSearch.Evolver(img, gmask, pop_size=args.pop)
ee.run(args.generations, checkpoint=args.checkpointfile)
seg = Segmentors.algoFromParams(ee.hof[0])
mask = seg.evaluate(img)
imageio.imwrite(args.outputfolder+"file.jpg", mask)
fitness,_ = Segmentors.FitnessFunction(mask,gmask)
print(f"{fitness} {ee.hof[0]}")
NUM_GENERATIONS = args.num_gen
NUM_TRIALS = args.num_trials
POP_SIZE = args.pop_size
print("Running {} Dataset".format(ds_name))
print("GA running for {} generations with population size of {}".format(
NUM_GENERATIONS, POP_SIZE))
print("Size of dataset: {}".format(len(X)))
print("Size of training set: {}".format(len(pipeline_dataset.training_set.y)))
print("Size of testing set: {}".format(len(pipeline_dataset.testing_set.y)))
print("\n")
# Initialize Algorithm Space and Workflow
Classifier.use_tutorial_space()
# Check algorithm space
algorithm_space = Classifier.algorithmspace
print("Algorithm Space: ")
print(list(algorithm_space.keys()))
print("\n")
workflow.addalgos([Classifier, ClassifierFitness])
wf = workflow()
for i in range(NUM_TRIALS):
print("Running trial number {}".format(i))
my_evolver = GeneticSearch.Evolver(workflow,
pipeline_dataset,
pop_size=POP_SIZE)
my_evolver.run(ngen=NUM_GENERATIONS, print_raw_data=True)
def test_make_toolbox():
"""Unit test for makeToolbox function. Checks that a toolbox of the
correct size was made."""
assert GeneticSearch.makeToolbox(
10, Segmentors.segmentor).population.keywords['n'] == 10
while download_status != 0:
print("Download failed retrying in", download_delay, "second(s)")
time.sleep(download_delay)
download_status = os.system(get_rgb)
download_status = download_status + os.system(get_label)
# Load the images
img = imageio.imread(rgb_filename)
gmask = imageio.imread(label_filename)
# Only needed on some images
# Convert the RGB 3-channel image into a 1-channel image
# gmask = (np.sum(gmask, axis=2) > 0)
# Create an evolver
my_evolver = GeneticSearch.Evolver(img, gmask, pop_size=pop_size)
# Conduct the genetic search
population = None
for i in range(num_gen):
# if population is uninitialized
if population is None:
population = my_evolver.run(ngen=1)
else:
# Simulate a generation and store population in population variable
population = my_evolver.run(ngen=1, population=population)
# Take the best segmentor from the hof and use it to segment the rgb image
seg = Segmentors.algoFromParams(my_evolver.hof[0])
# datasets[i] = helpers.generate_train_test_set(
# temp.training_set.X, temp.training_set.y
# )
datasets[i] = helpers.generate_train_test_set(ds.X, ds.y)
datasets[i].name = ds.name
temp = helpers.generate_train_test_set(ds.X, ds.y, random_state=31)
validation_sets.append(temp)
NUM_GENERATIONS = args.num_gen
POP_SIZE = args.pop_size
hof_per_dataset = []
for ds in datasets:
print("Running {} Dataset".format(ds.name))
my_evolver = GeneticSearch.Evolver(workflow, ds, pop_size=POP_SIZE)
my_evolver.run(
ngen=NUM_GENERATIONS,
# print_fitness_to_file=True,
print_fitness_to_file=True,
print_fitness_filename="{}_fitness_{}.csv".format(
ds.name, args.filename_tail),
)
# Store the best solution found for each dataset
hof_per_dataset.append(my_evolver.hof)
top_n = 5
for i, hof in enumerate(hof_per_dataset):
top_inds = hof[:top_n]
print('----------------\n')
for ind in top_inds:
def update_code_display(n_intevals, currently_displayed_code):
if has_results():
return GeneticSearch.print_best_algorithm_code(best_ind["params"])
else:
return currently_displayed_code