def test__index_linear_reinsertion_nsga_constraints(self):
"""
1) Compares selection to a manually calculated solution.
2) Tests the selection of individuals for crossover to verify whether the mean of the number of violations is t1<=t0. If the lowest violations individuals are indeed being selected.
"""
print("_index_linear_reinsertion_nsga_constraints")
data = np.loadtxt(
os.path.join(
self.path_data,
"_index_linear_reinsertion_nsga_constraints_data.csv"),
delimiter=",",
skiprows=1,
) # violations crowd_dist fronts
index = np.loadtxt(
os.path.join(
self.path_data,
"_index_linear_reinsertion_nsga_constraints_index.csv"),
delimiter=",",
skiprows=1,
dtype=np.int32,
)
ix_sel = AlgNsga2._index_linear_reinsertion_nsga_constraints(
data[:, 0], data[:, 1], data[:, 2], 100)
self.assertTrue(len(set(ix_sel) - set(index)) == 0)
# Test with second data
data = np.loadtxt(
os.path.join(
self.path_data,
"_index_linear_reinsertion_nsga_constraints_data_2.csv"),
delimiter=",",
skiprows=1,
) # violations crowd_dist fronts
index = np.loadtxt(
os.path.join(
self.path_data,
"_index_linear_reinsertion_nsga_constraints_index_2.csv"),
delimiter=",",
skiprows=1,
dtype=np.int32,
) # All indexes in the array are acceptable
ix_sel = AlgNsga2._index_linear_reinsertion_nsga_constraints(
data[:, 0], data[:, 1], data[:, 2], 100)
self.assertTrue(len(set(ix_sel) - set(index)) == 0)
# Test 2 Verify if median is decreasing
n_tests = 2000 # Number of tests
n_ind = 200
for i in range(n_tests):
violations = np.random.randint(0, n_ind / 20, size=n_ind)
mean_violations = np.mean(violations)
crowd_dist = np.random.randint(0, 100, size=n_ind)
fronts = np.random.randint(0, 3, size=n_ind)
ix_sel = AlgNsga2._index_linear_reinsertion_nsga_constraints(
violations, crowd_dist, fronts,
len(fronts) // 2)
self.assertLess(np.mean(violations[ix_sel]), mean_violations)
def test_select_pop_by_index(self):
"""Tests if:
1)selection of pop by index is indeed reducing population violations data in n_tests.
2)Verify invalid number of batches
3)Verify invalid value of active genes
"""
# Number of genes
num_genes = int(25)
# Number of products
num_products = int(4)
# Number of Objectives
num_objectives = 2
# Start date of manufacturing
start_date = datetime.date(2016, 12, 1) # YYYY-MM-DD.
qc_max_months = 4 # Max number of months
# Number of Months
num_months = 36
num_fronts = 3 # Number of fronts created
myPlan = Planning(
num_genes,
num_products,
num_objectives,
start_date,
qc_max_months,
num_months,
num_fronts,
)
n_tests = 100 # Number of tests
for i in range(n_tests):
pop = load_obj(
os.path.join(self.path_data, "select_pop_by_index_pop.pkl"))
violations_copy = np.copy(pop.backlogs[:, 6])
crowding_dist_copy = np.copy(pop.crowding_dist)
fronts_copy = np.copy(pop.fronts)
mean_violations = np.mean(violations_copy)
ix_sel = AlgNsga2._index_linear_reinsertion_nsga_constraints(
violations_copy, crowding_dist_copy, fronts_copy,
len(fronts_copy) //
2) # Input Violations,Crowding Distance, Fronts
self.assertLess(
np.mean(violations_copy[ix_sel]), mean_violations
) # Verifies if selected indexes indeed reduce violations mean
myPlan.select_pop_by_index(pop, ix_sel)
self.assertLess(
np.mean(pop.backlogs[:, 6]), mean_violations
) # Verifies if selected individuals indeed reduce violations mean
for i in range(0, len(pop.products_raw)):
self.assertFalse(
any(pop.batches_raw[i][pop.masks[i]] ==
0)) # Verify invalid number of batches (0)
self.assertFalse(
np.sum(pop.masks[i][pop.genes_per_chromo[i]:]) > 0
) # Verify invalid value of active genes (If true Invalid bool after number of active genes.)
def test__fronts(self):
"""Tests _fronts, that classifies solutions(Each row is a solution with different objectives values) into pareto fronts.
1) Tests using data manually classified to 3 fronts.
2) Tests random values to verify if fronts are within expected range
"""
print("_fronts")
# Test 1
num_fronts = 3
objectives_fn = np.loadtxt(
os.path.join(self.path_data, "_fronts.csv"),
delimiter=",",
skiprows=1
) # Index(['f0', 'f1', 'f2', 'f3', 'front'], dtype='object')
front_calc = AlgNsga2._fronts(objectives_fn[:, 0:3], num_fronts)
self.assertEqual(front_calc.all(), objectives_fn[:, 4].all())
# Test 2
# Matrix of ones
objectives_fn = np.ones(shape=(100,
4)) # Index(['f0', 'f1', 'f2', 'f3'])
front_calc = AlgNsga2._fronts(objectives_fn, num_fronts)
self.assertFalse((any(front_calc >= num_fronts))
| (any(front_calc < 0)))
self.assertEqual(np.unique(front_calc)[0], 0)
# Matrix of zeros
objectives_fn = np.zeros(shape=(100,
4)) # Index(['f0', 'f1', 'f2', 'f3'])
front_calc = AlgNsga2._fronts(objectives_fn, num_fronts)
self.assertFalse((any(front_calc >= num_fronts))
| (any(front_calc < 0)))
self.assertEqual(np.unique(front_calc)[0], 0)
# Matrix of random
num_tests = 10
for i in range(num_tests):
objectives_fn = np.random.randint(0, 100, size=400).reshape(
-1, 4) # Index(['f0', 'f1', 'f2', 'f3'])
front_calc = AlgNsga2._fronts(objectives_fn, num_fronts)
self.assertFalse((any(front_calc >= num_fronts))
| (any(front_calc < 0)))
def test__crowding_distance(self):
"""Tests evaluation of crowding distance for a manually calculated crowding distance."""
print("_crowding_distance")
big_dummy = 10**5
objectives_fn = np.loadtxt(
os.path.join(self.path_data, "_crowding_distance.csv"),
delimiter=",",
skiprows=1
) # Index(['f0', 'f1', 'f2', 'f3', 'front','dcrowd'], dtype='object')
crowding_dist = AlgNsga2._crowding_distance(objectives_fn[:, 0:4],
objectives_fn[:, 4],
big_dummy)
delta = 0.0001
message = "First and second are not almost equal." # error message in case if test case got failed
self.assertAlmostEqual(crowding_dist.all(), objectives_fn[:, 5].all(),
None, message, delta)
def main(self, num_exec, num_chromossomes, num_geracoes, n_tour,
perc_crossover, pmut):
print("START Exec number:", num_exec)
t0 = perf_counter()
# 1) Random parent population is initialized with its attributes
pop = population.Population(
self.num_genes,
num_chromossomes,
self.num_products,
self.num_objectives,
self.start_date,
self.qc_max_months,
self.num_months,
)
# 1.1) Initializes class object for Offspring Population
# Number of chromossomes for crossover, guarantees an even number
n_parents = int(num_chromossomes * perc_crossover)
if n_parents % 2 == 1:
n_parents = n_parents + 1
pop_offspring = population.Population(
self.num_genes,
n_parents,
self.num_products,
self.num_objectives,
self.start_date,
self.qc_max_months,
self.num_months,
)
# 1.2) Creates start and end date from schedule assures only batches with End date<Last day of manufacturing
# 2) Is calculated along Step 1, Note that USP end dates are calculated, but not stored.
pop = self.calc_start_end(pop)
# 3)Calculate inventory levels and objectives
pop = self.calc_inventory_objectives(pop)
# 4)Front Classification
objectives_raw_copy = pop.objectives_raw.copy()
pop.fronts = AlgNsga2._fronts(objectives_raw_copy, self.num_fronts)
# 5) Crowding Distance
objectives_raw_copy = pop.objectives_raw.copy()
fronts_copy = pop.fronts.copy()
pop.crowding_dist = AlgNsga2._crowding_distance(
objectives_raw_copy, fronts_copy, self.big_dummy)
for i_gen in range(0, num_geracoes):
# print("Generation ", i_gen)
# 6)Selection for Crossover Tournament
backlogs_copy = pop.backlogs[:, 6].copy()
fronts_copy = pop.fronts.copy()
crowding_dist_copy = pop.crowding_dist.copy()
ix_to_crossover = self.tournament_restrictions(
fronts_copy, crowding_dist_copy, n_parents, n_tour,
backlogs_copy)
# 7)Crossover
# 7.1 Sorts Selected by number of genes
genes_per_chromo_copy = pop.genes_per_chromo.copy()
ix_to_crossover = ix_to_crossover[np.argsort(
genes_per_chromo_copy[ix_to_crossover])]
# 7.2 Creates a new population for offspring population crossover and calls uniform crossover
products_raw_copy = pop.products_raw.copy()
batches_raw_copy = pop.batches_raw.copy()
masks_copy = pop.masks.copy()
new_products, new_batches, new_mask = Crossovers._crossover_uniform(
products_raw_copy[ix_to_crossover],
batches_raw_copy[ix_to_crossover],
masks_copy[ix_to_crossover],
perc_crossover,
)
# 8)Mutation
new_products, new_batches, new_mask = self.mutation_processes(
new_products, new_batches, new_mask, pmut)
# 9)Aggregate batches with same product neighbours
new_products, new_batches, new_mask = self.fix_aggregation_batches(
new_products, new_batches, new_mask)
# 10) Merge populations Current and Offspring
pop_offspring.update_new_population(new_products, new_batches,
new_mask)
# 11) 2) Is calculated along Step 1, Note that USP end dates are calculated, but not stored.
pop_offspring = self.calc_start_end(pop_offspring)
# 12) 3)Calculate inventory levels and objectives
pop_offspring = self.calc_inventory_objectives(pop_offspring)
# 13) Merge Current Pop with Offspring
pop = Planning.merge_pop_with_offspring(pop, pop_offspring)
# 14) 4)Front Classification
objectives_raw_copy = pop.objectives_raw.copy()
pop.fronts = AlgNsga2._fronts(objectives_raw_copy, self.num_fronts)
# 15) 5) Crowding Distance
objectives_copy = pop.objectives_raw.copy()
fronts_copy = pop.fronts.copy()
pop.crowding_dist = AlgNsga2._crowding_distance(
objectives_copy, fronts_copy, self.big_dummy)
# 16) Linear Reinsertion
# 16.1) Selects indexes to maintain
# Calculates number of violated constraints
backlogs_copy = np.copy(pop.backlogs[:, 6])
crowding_dist_copy = np.copy(pop.crowding_dist)
fronts_copy = np.copy(pop.fronts)
ix_reinsert = AlgNsga2._index_linear_reinsertion_nsga_constraints(
backlogs_copy,
crowding_dist_copy,
fronts_copy,
num_chromossomes,
)
# 16.2) Remove non reinserted chromossomes from pop
ix_reinsert_copy = np.copy(ix_reinsert)
self.select_pop_by_index(pop, ix_reinsert_copy)
t1 = perf_counter()
print("Exec", num_exec, "Time", t1 - t0)
# gc.collect()
return pop
def run_parallel(numExec, numGenerations, maxWorkers):
"""Run GA using Multiprocessing.
Args:
numExec (int): Number of executions
numGenerations (int): Number of generations
maxWorkers (int): Number of workers
"""
# Parameters
n_exec_ite = range(0, numExec)
# Number of genes
num_genes = int(25)
# Number of products
num_products = int(4)
# Number of Objectives
num_objectives = 2
# Start date of manufacturing
start_date = datetime.date(2016, 12, 1) # YYYY-MM-DD.
qc_max_months = 4 # Max number of months
# Number of Months
num_months = 36
num_fronts = 3 # Number of fronts created
# Inversion val to convert maximization of throughput to minimization, using a value a little bit higher than the article max 630.4
ref_point = [2500, 2500]
volume_max = np.prod(ref_point) # Maximum Volume
# Variables
# Variant
var = "front_nsga,tour_vio,rein_vio,vio_back,calc_montecarlo"
# Number of Chromossomes
nc = [100]
ng = [numGenerations] # Number of Generations
# Number of tour
nt = [2]
# Crossover Probability
pcross = [0.5]
# Parameters for the mutation operator (pmutp,pposb,pnegb,pswap)
pmut = [(0.04, 0.61, 0.77, 0.47)]
output_data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "data/output_raw/"))
if os.path.exists(output_data_path) == False:
raise Exception(f"Could not find the path {output_data_path}, please modify the path.")
# List of variants
list_vars = list(product(*[nc, ng, nt, pcross, pmut]))
# Lists store results
result_execs = []
result_ids = []
times = []
for v_i in list_vars:
name_var = f"{var},{v_i[0]},{v_i[1]},{v_i[2]},{v_i[3]},{v_i[4]}"
# Creates a dummy pop with one chromossome to concatenate results
pop_main = population.Population(
num_genes,
1,
num_products,
num_objectives,
start_date,
qc_max_months,
num_months,
)
pop_main.name_variation = name_var
file_name = f"pop_{v_i[0]},{v_i[1]},{v_i[2]},{v_i[3]},{v_i[4]}.pkl"
export_obj(pop_main, os.path.join(output_data_path, file_name))
del pop_main # 1)Is it a best practice delete an object after exporting and then load, export and del again?
t0 = perf_counter()
with concurrent.futures.ProcessPoolExecutor(max_workers=maxWorkers) as executor:
for pop_exec in executor.map(
planning.Planning(
num_genes,
num_products,
num_objectives,
start_date,
qc_max_months,
num_months,
num_fronts,
).main,
n_exec_ite,
[v_i[0]] * numExec,
[v_i[1]] * numExec,
[v_i[2]] * numExec,
[v_i[3]] * numExec,
[v_i[4]] * numExec,
):
pop_main = load_obj(os.path.join(output_data_path, file_name))
print("In merge num_chromossomes", pop_main.num_chromossomes)
pop_main = planning.Planning.merge_pop_with_offspring(pop_main, pop_exec)
print("Out merge num_chromossomes", pop_main.num_chromossomes)
export_obj(pop_main, os.path.join(output_data_path, file_name))
del pop_main
del pop_exec
gc.collect()
pop_main = load_obj(os.path.join(output_data_path, file_name))
# Removes the first dummy one chromossome
print("Final Num Chromossomes", pop_main.fronts.shape)
planning.Planning.select_pop_by_index(pop_main, np.arange(1, pop_main.num_chromossomes))
# Front Classification
pop_main.fronts = AlgNsga2._fronts(pop_main.objectives_raw, num_fronts)
print("fronts out", pop_main.fronts)
# Select only front 0 with no violations or front 0
ix_vio = np.where(pop_main.backlogs[:, 6] == 0)[0]
ix_par = np.where(pop_main.fronts == 0)[0]
ix_pareto_novio = np.intersect1d(ix_vio, ix_par)
if len(ix_pareto_novio) > 0:
var = var + "metrics_front0_wo_vio"
print(
"Found Solutions without violations and in pareto front",
len(ix_pareto_novio),
ix_pareto_novio,
)
else:
print(
"No solution without violations and in front 0, passing all in front 0.",
ix_pareto_novio,
)
var = var + "metrics_front0_w_vio"
ix_pareto_novio = ix_par
print("Fronts In select by index", pop_main.fronts)
print("Backlog In select by index", pop_main.backlogs[:, 6])
planning.Planning.select_pop_by_index(pop_main, ix_pareto_novio)
print("Fronts out select by index", pop_main.fronts)
print("Backlog out select by index", pop_main.backlogs[:, 6])
print("Objectives before metrics_inversion_violations", pop_main.objectives_raw)
# Extract Metrics
r_exec, r_ind = pop_main.metrics_inversion_violations(
ref_point,
volume_max,
num_fronts,
0,
name_var,
pop_main.backlogs[:, 6],
)
result_execs.append(r_exec)
result_ids.append(r_ind[0]) # X
result_ids.append(r_ind[1]) # Y
print("Objectives after metrics_inversion_violations", pop_main.objectives_raw)
print("Backlog Out after metrics_inversion_violations", pop_main.backlogs[:, 6])
file_name = f"pop_{v_i[0]},{v_i[1]},{v_i[2]},{v_i[3]},{v_i[4]}.pkl"
export_obj(pop_main, os.path.join(output_data_path, file_name))
tf = perf_counter()
delta_t = tf - t0
print("Total time ", delta_t, "Per execution", delta_t / numExec)
times.append([v_i, delta_t, delta_t / numExec])
name_var = "v_0"
# name_var=f"exec{numExec}_chr{nc}_ger{ng}_tour{nt}_cross{pcross}_mut{pmut}"
# print(f"{tempo} tempo/exec{tempo/numExec}")
# Export times
export_list_to_csv(
times, os.path.join(output_data_path, name_var + "_results.csv"), methodexport="a"
)
# Export Pickle
export_obj(result_execs, os.path.join(output_data_path, name_var + "_exec.pkl"))
export_obj(result_ids, os.path.join(output_data_path, name_var + "_id.pkl"))
print("Finish")
def process_incomplete_populations(numGenerations):
"""Used to process runs where executions were not totally completed, generates a pareto front using all populations and finishes the results exportation as in the original run.py execution.
Args:
numGenerations (int): Number of generations
"""
num_fronts = 3 # Number of fronts created
# Inversion val to convert maximization of throughput to minimization, using a value a little bit higher than the article max 630.4
ref_point = [2500, 2500]
volume_max = np.prod(ref_point) # Maximum Volume
# Variables
# Variant
var = "front_nsga,tour_vio,rein_vio,vio_back,calc_montecarlo"
# Number of Chromossomes
nc = [100]
ng = [numGenerations] # Number of Generations
# Number of tour
nt = [2]
# Crossover Probability
# pcross = [0.11]
pcross = [0.5]
# Parameters for the mutation operator (pmutp,pposb,pnegb,pswap)
pmut = [(0.04, 0.61, 0.77, 0.47)]
output_data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "data/output_raw/"))
if os.path.exists(output_data_path) == False:
raise Exception(f"Could not find the path {output_data_path}, please modify the path.")
# List of variants
list_vars = list(product(*[nc, ng, nt, pcross, pmut]))
# Lists store results
result_execs = []
result_ids = []
times = []
for v_i in list_vars:
name_var = f"{var},{v_i[0]},{v_i[1]},{v_i[2]},{v_i[3]},{v_i[4]}"
file_name = f"pop_{v_i[0]},{v_i[1]},{v_i[2]},{v_i[3]},{v_i[4]}.pkl"
# Open Population
pop_main = load_obj(os.path.join(output_data_path, file_name))
# Removes the first dummy one chromossome
print("Num Chromossomes", pop_main.fronts.shape)
planning.Planning.select_pop_by_index(pop_main, np.arange(1, pop_main.num_chromossomes))
# Front Classification
pop_main.fronts = AlgNsga2._fronts(pop_main.objectives_raw, num_fronts)
print("fronts out", pop_main.fronts)
# Select only front 0 with no violations or front 0
ix_vio = np.where(pop_main.backlogs[:, 6] == 0)[0]
ix_par = np.where(pop_main.fronts == 0)[0]
ix_pareto_novio = np.intersect1d(ix_vio, ix_par)
if len(ix_pareto_novio) > 0:
var = var + "metrics_front0_wo_vio"
print(
"Found Solutions without violations and in pareto front",
len(ix_pareto_novio),
ix_pareto_novio,
)
else:
print(
"No solution without violations and in front 0, passing all in front 0.",
ix_pareto_novio,
)
var = var + "metrics_front0_w_vio"
ix_pareto_novio = ix_par
print("Fronts In select by index", pop_main.fronts)
print("Backlog In select by index", pop_main.backlogs[:, 6])
planning.Planning.select_pop_by_index(pop_main, ix_pareto_novio)
print("Fronts out select by index", pop_main.fronts)
print("Backlog out select by index", pop_main.backlogs[:, 6])
print("Objectives before metrics_inversion_violations", pop_main.objectives_raw)
# Extract Metrics
r_exec, r_ind = pop_main.metrics_inversion_violations(
ref_point,
volume_max,
num_fronts,
0,
name_var,
pop_main.backlogs[:, 6],
)
result_execs.append(r_exec)
result_ids.append(r_ind[0]) # X
result_ids.append(r_ind[1]) # Y
print("Objectives after metrics_inversion_violations", pop_main.objectives_raw)
print("Backlog Out after metrics_inversion_violations", pop_main.backlogs[:, 6])
file_name = f"pop_{v_i[0]},{v_i[1]},{v_i[2]},{v_i[3]},{v_i[4]}.pkl"
export_obj(pop_main, os.path.join(output_data_path, file_name))
times.append([v_i, 0, 0])
name_var = "v_0"
# name_var=f"exec{numExec}_chr{nc}_ger{ng}_tour{nt}_cross{pcross}_mut{pmut}"
# print(f"{tempo} tempo/exec{tempo/numExec}")
# Export times
export_list_to_csv(
times, os.path.join(output_data_path, name_var + "_results.csv"), methodexport="a"
)
# Export Pickle
export_obj(result_execs, os.path.join(output_data_path, name_var + "_exec.pkl"))
export_obj(result_ids, os.path.join(output_data_path, name_var + "_id.pkl"))
print("Finish")