def alg(fixed_schedule_part, initial_schedule, current_time=0.0, initial_population=None, only_new_pops=False):
n = alg_params["n"]
### generate heft_based population
init_ind_count = int(n * init_sched_percent)
heft_particle = initial_schedule if isinstance(initial_schedule, (CompoundParticle, GsaCompoundParticle)) \
else generate_func(wf, rm, estimator, initial_schedule, fixed_schedule_part, current_time)
init_arr = [deepcopy(heft_particle) for _ in range(init_ind_count)]
generated_arr = [generate_func(wf, rm, estimator,
schedule=None,
fixed_schedule_part=fixed_schedule_part,
current_time=current_time)
for _ in range(n - init_ind_count)]
heft_based_population = init_arr + generated_arr
### generate new population
random_population = [generate_func(wf, rm, estimator,
schedule=None,
fixed_schedule_part=fixed_schedule_part,
current_time=current_time)
for _ in range(n)]
populations = {
"inherited": initial_population,
"heft_based": heft_based_population,
"random": random_population
}
if "inherited" in populations and(populations["inherited"] is None or len(populations["inherited"])):
del populations["inherited"]
def migration(populations, k):
pops = [pop for name, pop in sorted(populations.items(), key=lambda x: x[0])]
migRing(pops, k, selection=emigrant_selection)
task_map = {task.id: task for task in wf.get_all_unique_tasks()}
node_map = {node.name: node for node in rm.get_nodes()}
schedule_builder = ParticleScheduleBuilder(wf, rm, estimator,
task_map, node_map,
fixed_schedule_part)
pf_schedule = partial(schedule_builder, current_time=current_time)
toolbox = Toolbox()
toolbox.register("run_alg", algorithm(pf_schedule=pf_schedule, generate_=lambda n: None, **alg_params))
toolbox.register("migration", migration)
lb, st = deepcopy(log_book), deepcopy(stats)
pop, logbook, best = run_mpga(toolbox=toolbox, logbook=lb, stats=st, initial_populations=populations, **alg_params)
resulted_schedule = pf_schedule(best)
result = (best, pop, resulted_schedule, None), logbook
return result
def run_dcga(wf, estimator, rm, heft_mapping, heft_ordering, **params):
cxpb = params["cxpb"]#0.9
mutpb = params["mutpb"]#0.9
ngen = params["ngen"]#100
pop_size = params["pop_size"]#100
ctx = {'env': Env(wf, rm, estimator)}
toolbox = Toolbox()
toolbox.register("select", tools.selTournament, tournsize=2)
toolbox.register("mate", _mate, ctx)
toolbox.register("mutate", _mutate, ctx)
toolbox.register("evaluate", lambda x: [fitness_mapping_and_ordering(ctx, {MAPPING_SPECIE: x[0], ORDERING_SPECIE: x[1]})])
# heft_mapping = extract_mapping_from_ga_file("../../temp/heft_etalon_full_tr100_m100.json", rm)
pop_mapping = mapping_heft_based_initialize(ctx, pop_size, heft_mapping, 3)
pop_ordering = ordering_heft_based_initialize(ctx, pop_size, heft_ordering, 3)
pop = [ListBasedIndividual(el) for el in zip(pop_mapping, pop_ordering)]
for p in pop:
p.fitness = Fitness(0)
stat = tools.Statistics(key=lambda x: x.fitness)
stat.register("solsstat", lambda pop: [{"best": numpy.max(pop).values[0]}])
final_pop, logbook = deap.algorithms.eaSimple(pop, toolbox, cxpb, mutpb, ngen, stat)
best = max(final_pop, key=lambda x: x.fitness)
return best.fitness.values[0], logbook
def _initialize_toolbox(self, toolbox: base.Toolbox):
toolbox.register(
'individual',
TrapDistribution,
constrained_nodes = self._unique_constrained_nodes,
trap_amount = self._trap_amount,
random_initialization = True,
)
toolbox.register('evaluate', lambda d: self._constraint_set.score(d))
toolbox.register('mate', mate_distributions, distributor=self)
toolbox.register('mutate', mutate_trap_distribution)
def toolbox(self, transfer):
_wf, rm, estimator = self.env()
estimator.transfer_time = transfer
heft_schedule = self.heft_schedule()
heft_particle = generate(_wf, rm, estimator, heft_schedule)
heft_gen = lambda n: ([
deepcopy(heft_particle)
if random.random() > 1.00 else generate(_wf, rm, estimator)
for _ in range(n - 1)
] + [deepcopy(heft_particle)])
#heft_gen = lambda n: ([deepcopy(heft_particle) if random.random() > 1.00 else generate(_wf, rm, estimator) for _ in range(n)])
def componoud_update(w, c1, c2, p, best, pop, min=-1, max=1):
#doMap = random.random()
#if doMap < 0.5:
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
ordering_update(w,
c1,
c2,
p.ordering,
best.ordering,
pop,
min=min,
max=max)
toolbox = Toolbox()
toolbox.register("population", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator)
toolbox.register("update", componoud_update)
return toolbox
def toolbox(self, mapMatrix, rankList, ordFilter):
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
heft_particle = rd_order.generate(_wf, rm, estimator, mapMatrix,
rankList, ordFilter, heft_schedule)
heft_gen = lambda n: [
deepcopy(heft_particle)
if random.random() > 1.00 else rd_order.generate(
_wf, rm, estimator, mapMatrix, rankList, ordFilter)
for _ in range(n)
]
#def componoud_update(w, c1, c2, p, best, pop, g):
def componoud_update(w, c1, c2, p, best, pop):
#if g%2 == 0:
rd_map.update(w, c1, c2, p.mapping, best.mapping, pop)
#else:
rd_order.ordering_update(w, c1, c2, p.ordering, best.ordering, pop)
toolbox = Toolbox()
toolbox.register("population", heft_gen)
toolbox.register("fitness", rd_order.fitness, _wf, rm, estimator)
toolbox.register("update", componoud_update)
return toolbox
def toolbox(self, mapMatrix, rankList, ordFilter):
_wf, rm, estimator = self.env()
estimator.transfer_time = 500
heft_schedule = self.heft_schedule()
heft_particle = generate(_wf, rm, estimator, mapMatrix, rankList,
ordFilter, heft_schedule)
heft_gen = lambda n: ([
deepcopy(heft_particle) if random.random() > 1.00 else generate(
_wf, rm, estimator, mapMatrix, rankList, ordFilter)
for _ in range(n - 1)
] + [deepcopy(heft_particle)])
#heft_gen = lambda n: [deepcopy(heft_particle) if random.random() > 1.00 else generate(_wf, rm, estimator, mapMatrix, rankList, ordFilter) for _ in range(n)]
def componoud_update(w, c1, c2, p, best, pop):
#doMap = random.random()
#if doMap < 0.1:
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
#else:
ordering_update(w, c1, c2, p.ordering, best.ordering, pop)
toolbox = Toolbox()
toolbox.register("population", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator)
toolbox.register("update", componoud_update)
return toolbox
def create_pso_alg(pf_schedule, generate_, **params):
def fit_converter(func):
def wrap(*args, **kwargs):
x = func(*args, **kwargs)
m = Utility.makespan(x)
return FitnessStd(values=(m, 0.0))
return wrap
def componoud_update(w, c1, c2, p, best, pop, min=-1, max=1):
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
ordering_update(w,
c1,
c2,
p.ordering,
best.ordering,
pop,
min=min,
max=max)
toolbox = Toolbox()
toolbox.register("population", generate_)
toolbox.register("fitness", fit_converter(pf_schedule))
toolbox.register("update", componoud_update)
pso_alg = partial(run_pso, toolbox=toolbox, **params)
return pso_alg
def toolbox(self):
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
heft_particle = om_order.generate(_wf, rm, estimator, heft_schedule)
heft_gen = lambda n: [
deepcopy(heft_particle)
if random.random() > 1.00 else om_order.generate(
_wf, rm, estimator) for _ in range(n)
]
def componoud_update(w, c1, c2, p, best, pop, min=-1, max=1):
om_map.update(w, c1, c2, p.mapping, best.mapping, pop)
om_order.ordering_update(w,
c1,
c2,
p.ordering,
best.ordering,
pop,
min=min,
max=max)
toolbox = Toolbox()
toolbox.register("population", heft_gen)
toolbox.register("fitness", om_order.fitness, _wf, rm, estimator)
toolbox.register("update", componoud_update)
return toolbox
def _initialize_toolbox(self, toolbox: base.Toolbox):
toolbox.register(
'individual',
DistributionDelta,
origin = self._origin_trap_distribution,
added_nodes = self._added,
removed_node_indexes = self._removed_trap_indexes,
max_trap_difference = self._max_trap_delta,
trap_amount_delta = self.trap_amount - self._origin_trap_distribution.trap_amount,
)
toolbox.register('evaluate', lambda d: self._constraint_set.score(d.trap_distribution))
toolbox.register('mate', mate_distribution_deltas)
toolbox.register('mutate', mutate_distribution_delta)
def create_set_ind(cls, toolbox: base.Toolbox, fitness_function: callable,
possible_values: list):
"""
Method to create the genotype of the individuals represented by a Set from the list of
possible values received as an argument.
Parameters
----------
toolbox: base.Toolbox
DEAP Toolbox instance.
fitness_function: callable
DEAP fitness function.
possible_values: list
List of possible values to insert into the individual.
"""
creator.create('Individual', set, fitness=fitness_function)
toolbox.register('attr_set',
initSetGenotype,
possible_values=list(possible_values))
toolbox.register('individual', tools.initIterate, creator.Individual,
toolbox.attr_set)
def create_permutation_ind(cls,
toolbox: base.Toolbox,
fitness_function: callable,
initial_values: list = None,
individual_size: int = None):
"""
Method that allows to create and register (following the guidelines defined in DEAP) the
genotype of the individuals (registered as 'Individual') and the generating function of
individuals (registered as 'individual').
Parameters
----------
toolbox: base.Toolbox
DEAP Toolbox instance.
fitness_function: callable
DEAP fitness function.
initial_values: list, default=None
List of list of initial genotypes used for the creation of the initial population, this
allows incorporating a priori knowledge about better solutions and usually gives better
results than random initialisation of the genotypes.
If this parameter is not provided, it will be necessary to provide tha argument
individual_size.
individual_size: int, default=None
Size of the individual genotype.
If this parameter is not provided, it will be necessary to provide tha argument
individual_size.
Notes
-----
Parameters initial_values and individual_size cannot be provided at the same time.
"""
assert (initial_values is None or individual_size is None) and \
not (initial_values is None and individual_size is None), \
'Either the initial_values or individual_size must be provided.'
# Create from initial values
if initial_values is not None:
ind_generator = lambda initial_values: initial_values[
random.randint(0,
len(initial_values) - 1)]
toolbox.register('attr_permutation', ind_generator, initial_values)
# Create randomly
else:
toolbox.register('attr_permutation', random.sample,
range(individual_size), individual_size)
creator.create('Individual', list, fitness=fitness_function)
toolbox.register('individual', tools.initIterate, creator.Individual,
toolbox.attr_permutation)
def run_dcga(wf, estimator, rm, heft_mapping, heft_ordering, **params):
cxpb = params["cxpb"] #0.9
mutpb = params["mutpb"] #0.9
ngen = params["ngen"] #100
pop_size = params["pop_size"] #100
ctx = {'env': Env(wf, rm, estimator)}
toolbox = Toolbox()
toolbox.register("select", tools.selTournament, tournsize=2)
toolbox.register("mate", _mate, ctx)
toolbox.register("mutate", _mutate, ctx)
toolbox.register(
"evaluate", lambda x: [
fitness_mapping_and_ordering(ctx, {
MAPPING_SPECIE: x[0],
ORDERING_SPECIE: x[1]
})
])
# heft_mapping = extract_mapping_from_ga_file("../../temp/heft_etalon_full_tr100_m100.json", rm)
pop_mapping = mapping_heft_based_initialize(ctx, pop_size, heft_mapping, 3)
pop_ordering = ordering_heft_based_initialize(ctx, pop_size, heft_ordering,
3)
pop = [ListBasedIndividual(el) for el in zip(pop_mapping, pop_ordering)]
for p in pop:
p.fitness = Fitness(0)
stat = tools.Statistics(key=lambda x: x.fitness)
stat.register("solsstat", lambda pop: [{"best": numpy.max(pop).values[0]}])
final_pop, logbook = deap.algorithms.eaSimple(pop, toolbox, cxpb, mutpb,
ngen, stat)
best = max(final_pop, key=lambda x: x.fitness)
return best.fitness.values[0], logbook
def alg(fixed_schedule_part, initial_schedule, current_time=0.0):
def generate_(n):
init_ind_count = int(n*init_sched_percent)
res = []
if initial_schedule is not None and init_ind_count > 0:
heft_particle = pso_generate(wf, rm, estimator, initial_schedule)
init_arr = [deepcopy(heft_particle) for _ in range(init_ind_count)]
res = res + init_arr
if n - init_ind_count > 0:
generated_arr = [pso_generate(wf, rm, estimator,
schedule=None,
fixed_schedule_part=fixed_schedule_part,
current_time=current_time)
for _ in range(n - init_ind_count)]
res = res + generated_arr
return res
def fit_converter(func):
def wrap(*args, **kwargs):
x = func(*args, **kwargs)
m = Utility.makespan(x)
return FitnessStd(values=(m, 0.0))
return wrap
def componoud_update(w, c1, c2, p, best, pop, min=-1, max=1):
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
ordering_update(w, c1, c2, p.ordering, best.ordering, pop, min=min, max=max)
task_map = {task.id: task for task in wf.get_all_unique_tasks()}
node_map = {node.name: node for node in rm.get_nodes()}
schedule_builder = ParticleScheduleBuilder(wf, rm, estimator,
task_map, node_map,
fixed_schedule_part)
pf_schedule = partial(schedule_builder, current_time=current_time)
toolbox = Toolbox()
toolbox.register("population", generate_)
toolbox.register("fitness", fit_converter(pf_schedule))
toolbox.register("update", componoud_update)
pop, logbook, best = run_pso(toolbox=toolbox, **params)
resulted_schedule = pf_schedule(best)
result = (best, pop, resulted_schedule, None), logbook
return result
def toolbox(self):
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
heft_particle = generate(_wf, rm, estimator, heft_schedule)
heft_gen = lambda n: [deepcopy(heft_particle) if random.random() > 1.00 else generate(_wf, rm, estimator) for _ in range(n)]
def componoud_update(w, c1, c2, p, best, pop, min=-1, max=1):
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
ordering_update(w, c1, c2, p.ordering, best.ordering, pop, min=min, max=max)
toolbox = Toolbox()
toolbox.register("population", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator)
toolbox.register("update", componoud_update)
return toolbox
def create_pso_alg(pf_schedule, generate_, **params):
def fit_converter(func):
def wrap(*args, **kwargs):
x = func(*args, **kwargs)
m = Utility.makespan(x)
return FitnessStd(values=(m, 0.0))
return wrap
def componoud_update(w, c1, c2, p, best, pop, min=-1, max=1):
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
ordering_update(w, c1, c2, p.ordering, best.ordering, pop, min=min, max=max)
toolbox = Toolbox()
toolbox.register("population", generate_)
toolbox.register("fitness", fit_converter(pf_schedule))
toolbox.register("update", componoud_update)
pso_alg = partial(run_pso, toolbox=toolbox, **params)
return pso_alg
def toolbox(self, rankList):
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
heft_particle = generate(_wf, rm, estimator, rankList, heft_schedule)
#heft_gen = lambda n: ([deepcopy(heft_particle) if random.random() > 1.00 else generate(_wf, rm, estimator, rankList) for _ in range(n-1)] + [deepcopy(heft_particle)])
heft_gen = lambda n: ([
deepcopy(heft_particle) if random.random() > 1.00 else generate(
_wf, rm, estimator, rankList) for _ in range(n)
])
def componoud_update(w, c1, c2, p, best, pop):
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
ordering_update(w, c1, c2, p.ordering, best.ordering, pop)
toolbox = Toolbox()
toolbox.register("population", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator)
toolbox.register("update", componoud_update)
return toolbox
def register_crossover_operator(self, toolbox: Toolbox) -> None:
# the crossover between individuals is a uniform crossover
# that means each test case has 50-50 probability of ending up in each of the new individuals
toolbox.register("mate", tools.cxUniform, indpb=0.5)
def register_mutation_operator(self, toolbox: Toolbox) -> None:
toolbox.register("mutate", self.sapienz_mut_suite, indpb=0.5)
def alg(fixed_schedule_part,
initial_schedule,
current_time=0.0,
initial_population=None,
only_new_pops=False):
n = alg_params["n"]
### generate heft_based population
init_ind_count = int(n * init_sched_percent)
heft_particle = initial_schedule if isinstance(initial_schedule, (CompoundParticle, GsaCompoundParticle)) \
else generate_func(wf, rm, estimator, initial_schedule, fixed_schedule_part, current_time)
init_arr = [deepcopy(heft_particle) for _ in range(init_ind_count)]
generated_arr = [
generate_func(wf,
rm,
estimator,
schedule=None,
fixed_schedule_part=fixed_schedule_part,
current_time=current_time)
for _ in range(n - init_ind_count)
]
heft_based_population = init_arr + generated_arr
### generate new population
random_population = [
generate_func(wf,
rm,
estimator,
schedule=None,
fixed_schedule_part=fixed_schedule_part,
current_time=current_time) for _ in range(n)
]
populations = {
"inherited": initial_population,
"heft_based": heft_based_population,
"random": random_population
}
if "inherited" in populations and (populations["inherited"] is None
or len(populations["inherited"])):
del populations["inherited"]
def migration(populations, k):
pops = [
pop for name, pop in sorted(populations.items(),
key=lambda x: x[0])
]
migRing(pops, k, selection=emigrant_selection)
task_map = {task.id: task for task in wf.get_all_unique_tasks()}
node_map = {node.name: node for node in rm.get_nodes()}
schedule_builder = ParticleScheduleBuilder(wf, rm, estimator, task_map,
node_map,
fixed_schedule_part)
pf_schedule = partial(schedule_builder, current_time=current_time)
toolbox = Toolbox()
toolbox.register(
"run_alg",
algorithm(pf_schedule=pf_schedule,
generate_=lambda n: None,
**alg_params))
toolbox.register("migration", migration)
lb, st = deepcopy(log_book), deepcopy(stats)
pop, logbook, best = run_mpga(toolbox=toolbox,
logbook=lb,
stats=st,
initial_populations=populations,
**alg_params)
resulted_schedule = pf_schedule(best)
result = (best, pop, resulted_schedule, None), logbook
return result
def alg(fixed_schedule_part,
initial_schedule,
current_time=0.0,
initial_population=None):
ga_functions = GAFunctions2(wf, rm, estimator)
generate = partial(ga_generate,
ga_functions=ga_functions,
fixed_schedule_part=fixed_schedule_part,
current_time=current_time,
init_sched_percent=init_sched_percent,
initial_schedule=initial_schedule)
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register(
"evaluate",
fit_converter(
ga_functions.build_fitness(fixed_schedule_part, current_time)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox, **alg_params)
resulted_schedule = ga_functions.build_schedule(
best, fixed_schedule_part, current_time)
result = (best, pop, resulted_schedule, None), logbook
return result
def register_selection_operator(self, toolbox: Toolbox) -> None:
toolbox.register("select", tools.selRoulette)
def create_gsa_alg(pf_schedule, generate_, **params):
def fit_converter(func):
def wrap(*args, **kwargs):
x = func(*args, **kwargs)
m = Utility.makespan(x)
return FitnessStd(values=(m, 0.0))
return wrap
def compound_force(p, pop, kbest, G):
mapping_force = force(p.mapping, (p.mapping for p in pop), kbest, G)
ordering_force = force(p.ordering, (p.ordering for p in pop), kbest, G)
return (mapping_force, ordering_force)
def compound_update(w, c, p, min=-1, max=1):
gsa_mapping_update(w, c, p.mapping)
gsa_ordering_update(w, c, p.ordering, min, max)
pass
W, C = params["w"], params["w"]
if "generations_count_before_merge" in params and "generations_count_after_merge" in params:
all_iterations_count = int(
params["generations_count_before_merge"]) + int(
params["generations_count_after_merge"])
else:
all_iterations_count = None
toolbox = Toolbox()
toolbox.register("generate", generate_)
toolbox.register("fitness", fit_converter(pf_schedule))
toolbox.register("estimate_force", compound_force)
toolbox.register("update", compound_update, W, C)
toolbox.register("G", partial(G, all_iter_number=all_iterations_count))
toolbox.register("kbest",
partial(Kbest, all_iter_number=all_iterations_count))
pso_alg = partial(run_gsa, toolbox=toolbox, **params)
return pso_alg
def register_mutation_operator(self, toolbox: Toolbox) -> None:
evlutiz_mutation = EvolutizMutation(self.evolutiz_connector)
toolbox.register("mutate", evlutiz_mutation.mutation)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_mapping.velocity = Velocity({})
heft_gen = lambda n: [deepcopy(heft_mapping) if random.random() > 1.0 else generate(_wf, rm, estimator, 1)[0] for _ in range(n)]
W, C1, C2 = 0.9, 0.6, 0.2
GEN, N = 500, 30
toolbox = Toolbox()
toolbox.register("population", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("update", update)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
def do_exp():
def create_gsa_alg(pf_schedule, generate_, **params):
def fit_converter(func):
def wrap(*args, **kwargs):
x = func(*args, **kwargs)
m = Utility.makespan(x)
return FitnessStd(values=(m, 0.0))
return wrap
def compound_force(p, pop, kbest, G):
mapping_force = force(p.mapping, (p.mapping for p in pop), kbest, G)
ordering_force = force(p.ordering, (p.ordering for p in pop), kbest, G)
return (mapping_force, ordering_force)
def compound_update(w, c, p, min=-1, max=1):
gsa_mapping_update(w, c, p.mapping)
gsa_ordering_update(w, c, p.ordering, min, max)
pass
W, C = params["w"], params["w"]
if "generations_count_before_merge" in params and "generations_count_after_merge" in params:
all_iterations_count = int(params["generations_count_before_merge"]) + int(params["generations_count_after_merge"])
else:
all_iterations_count = None
toolbox = Toolbox()
toolbox.register("generate", generate_)
toolbox.register("fitness", fit_converter(pf_schedule))
toolbox.register("estimate_force", compound_force)
toolbox.register("update", compound_update, W, C)
toolbox.register("G", partial(G, all_iter_number=all_iterations_count))
toolbox.register("kbest", partial(Kbest, all_iter_number=all_iterations_count))
pso_alg = partial(run_gsa, toolbox=toolbox, **params)
return pso_alg
def do_exp(wf_name):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate, ga_functions=ga_functions,
fixed_schedule_part=empty_fixed_schedule_part,
current_time=0.0, init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0)
Utility.validate_static_schedule(_wf, resulted_schedule)
ga_makespan = Utility.makespan(resulted_schedule)
return ga_makespan
def alg(fixed_schedule_part, initial_schedule, current_time=0.0, initial_population=None):
ga_functions = GAFunctions2(wf, rm, estimator)
generate = partial(ga_generate, ga_functions=ga_functions,
fixed_schedule_part=fixed_schedule_part,
current_time=current_time, init_sched_percent=init_sched_percent,
initial_schedule=initial_schedule)
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(fixed_schedule_part, current_time)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox, **alg_params)
resulted_schedule = ga_functions.build_schedule(best, fixed_schedule_part, current_time)
result = (best, pop, resulted_schedule, None), logbook
return result
def compound_force(p, pop, kbest, G):
mapping_force = force(p.mapping, (p.mapping for p in pop), kbest, G)
ordering_force = force(p.ordering, (p.ordering for p in pop), kbest, G)
return (mapping_force, ordering_force)
def compound_update(w, c, p, min=-1, max=1):
mapping_update(w, c, p.mapping)
ordering_update(w, c, p.ordering, min, max)
pass
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator)
toolbox.register("estimate_force", compound_force)
toolbox.register("update", compound_update, W, C)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
stats = Statistics()
stats.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
stats.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
stats.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
stats.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ["gen", "G", "kbest"] + stats.fields
from heft.experiments.cga.utilities.common import repeat
_wf = wf("Montage_40")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda n: [deepcopy(heft_mapping) if random.random() > 1.0 else generate(_wf, rm, estimator, 1)[0] for _ in range(n)]
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix)
toolbox.register("velocity_and_position", velocity_and_position, beta=0.0)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
stats = Statistics()
stats.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
stats.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
stats.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
stats.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
solution = construct_solution(position, ordering)
sched = build_schedule(_wf, estimator, rm, solution)
makespan = Utility.makespan(sched)
## TODO: make a real estimation later
fit = FitnessStd(values=(makespan, 0.0))
## TODO: make a normal multi-objective fitness estimation
fit.mofit = makespan
return fit
return basefitness(_wf, rm, estimator, solution)
toolbox = Toolbox()
# common functions
toolbox.register("map", map)
toolbox.register("clone", deepcopy)
toolbox.register("population", population)
toolbox.register("fitness", fitness)
# pso functions
toolbox.register("update", update)
# ga functions
toolbox.register("mutate", mutate)
toolbox.register("mate", mate)
toolbox.register("select", tools.selNSGA2)
ga = partial(run_nsga2,
toolbox=toolbox,
logbook=None,
stats=None,
def toolbox(self):
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
heft_particle = generate(_wf, rm, estimator, heft_schedule)
heft_gen = lambda n: [
deepcopy(heft_particle)
if random.random() > 1.00 else generate(_wf, rm, estimator)
for _ in range(n)
]
def componoud_update(w, c1, c2, p, best, pop, min=-1, max=1):
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
ordering_update(w,
c1,
c2,
p.ordering,
best.ordering,
pop,
min=min,
max=max)
def compound_force_vector_matrix():
raise NotImplementedError()
def compound_velocity_and_postion():
raise NotImplementedError()
toolbox = Toolbox()
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", compound_force_vector_matrix)
toolbox.register("velocity_and_position",
compound_velocity_and_postion,
beta=0.0)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
return toolbox
def test_fixed_ordering(self):
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda: heft_mapping if random.random(
) > 0.95 else generate(_wf, rm, estimator)
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix, rm)
toolbox.register("velocity_and_position", velocity_and_position, _wf,
rm, estimator)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
statistics = Statistics()
statistics.register(
"min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
statistics.register(
"avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
statistics.register(
"max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
statistics.register(
"std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
pop_size = 100
iter_number = 100
kbest = pop_size
ginit = 5
final_pop = run_gsa(toolbox, statistics, logbook, pop_size,
iter_number, kbest, ginit)
best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit)
solution = {
MAPPING_SPECIE: list(zip(sorted_tasks, best)),
ORDERING_SPECIE: sorted_tasks
}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def compound_force(p, pop, kbest, G):
mapping_force = force(p.mapping, (p.mapping for p in pop), kbest, G)
ordering_force = force(p.ordering, (p.ordering for p in pop), kbest, G)
return (mapping_force, ordering_force)
def compound_update(w, c, p, min=-1, max=1):
mapping_update(w, c, p.mapping)
ordering_update(w, c, p.ordering, min, max)
pass
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator)
toolbox.register("estimate_force", compound_force)
toolbox.register("update", compound_update, W, C)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
stats = Statistics()
stats.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
stats.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
stats.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
stats.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ["gen", "G", "kbest"] + stats.fields
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_mapping.velocity = MappingParticle.Velocity({})
heft_gen = lambda n: [
deepcopy(heft_mapping)
if random.random() > 1.0 else generate(_wf, rm, estimator, 1)[0]
for _ in range(n)
]
W, C1, C2 = 0.1, 0.6, 0.2
GEN, N = 300, 50
toolbox = Toolbox()
toolbox.register("population", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("update", update)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
def do_exp():
def test_fixed_ordering(self):
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda: heft_mapping if random.random() > 0.95 else generate(_wf, rm, estimator)
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix, rm)
toolbox.register("velocity_and_position", velocity_and_position, _wf, rm, estimator)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
statistics = Statistics()
statistics.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
statistics.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
statistics.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
statistics.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
pop_size = 100
iter_number = 100
kbest = pop_size
ginit = 5
final_pop = run_gsa(toolbox, statistics, logbook, pop_size, iter_number, kbest, ginit)
best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit)
solution = {MAPPING_SPECIE: list(zip(sorted_tasks, best)), ORDERING_SPECIE: sorted_tasks}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
solution = construct_solution(position, ordering)
sched = build_schedule(_wf, estimator, rm, solution)
makespan = Utility.makespan(sched)
## TODO: make a real estimation later
fit = FitnessStd(values=(makespan, 0.0))
## TODO: make a normal multi-objective fitness estimation
fit.mofit = makespan
return fit
return basefitness(_wf, rm, estimator, solution)
toolbox = Toolbox()
# common functions
toolbox.register("map", map)
toolbox.register("clone", deepcopy)
toolbox.register("population", population)
toolbox.register("fitness", fitness)
# pso functions
toolbox.register("update", update)
# ga functions
toolbox.register("mutate", mutate)
toolbox.register("mate", mate)
toolbox.register("select", tools.selNSGA2)
ga = partial(run_nsga2, toolbox=toolbox, logbook=None, stats=None,
n=N, crossover_probability=CXPB, mutation_probability=MU)
pso = partial(run_pso, toolbox=toolbox, logbook=None, stats=None,
def provide_features() -> None:
# define subjects
features.provide('instrumented_subjects_path',
args.instrumented_subjects_path)
features.provide('continue_on_subject_failure',
args.continue_on_subject_failure)
features.provide('continue_on_repetition_failure',
args.continue_on_repetition_failure)
# define budget and repetitions
features.provide('repetitions', args.repetitions)
features.provide('repetitions_offset', args.repetitions_offset)
features.provide(
'budget_manager',
BudgetManager(time_budget=args.time_budget,
evaluations_budget=args.evaluations_budget))
# define devices configuration
features.provide('emulators_number', args.emulators_number)
features.provide('real_devices_number', args.real_devices_number)
features.provide('avd_series', args.avd_series)
features.provide('avd_manager', AvdManager())
features.provide('strategy', possible_strategies[args.strategy])
features.provide('test_suite_evaluator',
possible_test_suite_evaluators[args.evaluator])
# define test runner
test_runner = possible_test_runners[args.test_runner]
features.provide('test_runner', test_runner)
test_runner.register_minimum_api()
# define coverage fetcher app instrumentator
coverage_fetcher = possible_coverage_fetchers[args.coverage]
features.provide('coverage_fetcher', coverage_fetcher)
coverage_fetcher.register_app_instrumentator()
features.provide('emma_instrument_path', args.emma_instrument_path)
# define individual and population generators
features.provide('individual_generator',
possible_individual_generators[args.individual_generator])
features.provide('population_generator', PopulationGenerator)
# define extras
features.provide('verbose_level', args.verbose)
features.provide('write_logbook', args.write_logbook)
features.provide('write_history', args.write_history)
features.provide('write_hall_of_fame', args.write_hall_of_fame)
features.provide('compress', args.compress)
# singletons
toolbox = Toolbox()
toolbox.register("selectBest", tools.selBest)
features.provide('toolbox', toolbox)
features.provide('device_manager', DeviceManager())
features.provide('evaluate_scripts_folder_path',
args.evaluate_scripts_folder_path)
features.provide('evaluate_scripts_repetition_number',
args.evaluate_scripts_repetition_number)
features.provide('evaluate_scripts_algorithm_name',
args.evaluate_scripts_algorithm_name)
features.provide('skip_subject_if_logbook_in_results',
args.skip_subject_if_logbook_in_results)
def register_selection_operator(self, toolbox: Toolbox) -> None:
# self.toolbox.register("select", tools.selTournament, tournsize=5)
toolbox.register("select", tools.selNSGA2)
def toolbox(self):
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
heft_particle = generate(_wf, rm, estimator, heft_schedule)
heft_gen = lambda n: [deepcopy(heft_particle) if random.random() > 1.00 else generate(_wf, rm, estimator) for _ in range(n)]
def componoud_update(w, c1, c2, p, best, pop, min=-1, max=1):
mapping_update(w, c1, c2, p.mapping, best.mapping, pop)
ordering_update(w, c1, c2, p.ordering, best.ordering, pop, min=min, max=max)
def compound_force_vector_matrix():
raise NotImplementedError()
def compound_velocity_and_postion():
raise NotImplementedError()
toolbox = Toolbox()
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", compound_force_vector_matrix)
toolbox.register("velocity_and_position", compound_velocity_and_postion, beta=0.0)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
return toolbox
class GPHH(Solver, DeterministicPolicies):
T_domain = D
training_domains: List[T_domain]
verbose: bool
weight: int
pset: PrimitiveSet
toolbox: Toolbox
policy: DeterministicPolicies
params_gphh: ParametersGPHH
# policy: GPHHPolicy
evaluation_method: EvaluationGPHH
reference_permutations: Dict
permutation_distance: PermutationDistance
def __init__(
self,
training_domains: List[T_domain],
domain_model: SchedulingDomain,
weight: int,
# set_feature: Set[FeatureEnum]=None,
params_gphh: ParametersGPHH = ParametersGPHH.default(),
reference_permutations=None,
# reference_makespans=None,
training_domains_names=None,
verbose: bool = False,
):
self.training_domains = training_domains
self.domain_model = domain_model
self.params_gphh = params_gphh
# self.set_feature = set_feature
self.set_feature = self.params_gphh.set_feature
print("self.set_feature: ", self.set_feature)
print("Evaluation: ", self.params_gphh.evaluation)
# if set_feature is None:
# self.set_feature = {FeatureEnum.RESSOURCE_TOTAL,
# FeatureEnum.TASK_DURATION,
# FeatureEnum.N_SUCCESSORS,
# FeatureEnum.N_SUCCESSORS,
# FeatureEnum.RESSOURCE_AVG}
self.list_feature = list(self.set_feature)
self.verbose = verbose
self.pset = self.init_primitives(self.params_gphh.set_primitves)
self.weight = weight
self.evaluation_method = self.params_gphh.evaluation
self.initialize_cpm_data_for_training()
if self.evaluation_method == EvaluationGPHH.PERMUTATION_DISTANCE:
self.init_reference_permutations(
reference_permutations, training_domains_names
)
self.permutation_distance = self.params_gphh.permutation_distance
# if self.evaluation_method == EvaluationGPHH.SGS_DEVIATION:
# self.init_reference_makespans(reference_makespans, training_domains_names)
def init_reference_permutations(
self, reference_permutations={}, training_domains_names=[]
) -> None:
self.reference_permutations = {}
for i in range(len(self.training_domains)):
td = self.training_domains[i]
td_name = training_domains_names[i]
if td_name not in reference_permutations.keys():
# Run CP
td.set_inplace_environment(False)
solver = DOSolver(
policy_method_params=PolicyMethodParams(
base_policy_method=BasePolicyMethod.SGS_PRECEDENCE,
delta_index_freedom=0,
delta_time_freedom=0,
),
method=SolvingMethod.CP,
)
solver.solve(domain_factory=lambda: td)
raw_permutation = solver.best_solution.rcpsp_permutation
full_permutation = [x + 2 for x in raw_permutation]
full_permutation.insert(0, 1)
full_permutation.append(np.max(full_permutation) + 1)
print("full_perm: ", full_permutation)
self.reference_permutations[td] = full_permutation
else:
self.reference_permutations[td] = reference_permutations[td_name]
# def init_reference_makespans(self, reference_makespans={}, training_domains_names=[]) -> None:
# self.reference_makespans = {}
# for i in range(len(self.training_domains)):
# td = self.training_domains[i]
# td_name = training_domains_names[i]
# # for td in self.training_domains:
# print('td:',td)
# if td_name not in reference_makespans.keys():
# # Run CP
# td.set_inplace_environment(False)
# solver = DOSolver(policy_method_params=PolicyMethodParams(base_policy_method=BasePolicyMethod.FOLLOW_GANTT,
# delta_index_freedom=0,
# delta_time_freedom=0),
# method=SolvingMethod.CP)
# solver.solve(domain_factory=lambda: td)
#
# state = td.get_initial_state()
# states, actions, values = rollout_episode(domain=td,
# max_steps=1000,
# solver=solver,
# from_memory=state,
# verbose=False,
# outcome_formatter=lambda
# o: f'{o.observation} - cost: {o.value.cost:.2f}')
#
# makespan = sum([v.cost for v in values])
# self.reference_makespans[td] = makespan
# else:
# self.reference_makespans[td] = reference_makespans[td_name]
def _solve_domain(self, domain_factory: Callable[[], D]) -> None:
self.domain = domain_factory()
tournament_ratio = self.params_gphh.tournament_ratio
pop_size = self.params_gphh.pop_size
n_gen = self.params_gphh.n_gen
min_tree_depth = self.params_gphh.min_tree_depth
max_tree_depth = self.params_gphh.max_tree_depth
crossover_rate = self.params_gphh.crossover_rate
mutation_rate = self.params_gphh.mutation_rate
creator.create("FitnessMin", Fitness, weights=(self.weight,))
creator.create("Individual", PrimitiveTree, fitness=creator.FitnessMin)
self.toolbox = Toolbox()
self.toolbox.register(
"expr",
genHalfAndHalf,
pset=self.pset,
min_=min_tree_depth,
max_=max_tree_depth,
)
self.toolbox.register(
"individual", tools.initIterate, creator.Individual, self.toolbox.expr
)
self.toolbox.register(
"population", tools.initRepeat, list, self.toolbox.individual
)
self.toolbox.register("compile", gp.compile, pset=self.pset)
if self.evaluation_method == EvaluationGPHH.SGS:
self.toolbox.register(
"evaluate", self.evaluate_heuristic, domains=self.training_domains
)
# if self.evaluation_method == EvaluationGPHH.SGS_DEVIATION:
# self.toolbox.register("evaluate", self.evaluate_heuristic_sgs_deviation, domains=self.training_domains)
elif self.evaluation_method == EvaluationGPHH.PERMUTATION_DISTANCE:
self.toolbox.register(
"evaluate",
self.evaluate_heuristic_permutation,
domains=self.training_domains,
)
# self.toolbox.register("evaluate", self.evaluate_heuristic, domains=[self.training_domains[1]])
self.toolbox.register(
"select", tools.selTournament, tournsize=int(tournament_ratio * pop_size)
)
self.toolbox.register("mate", gp.cxOnePoint)
self.toolbox.register("expr_mut", gp.genFull, min_=0, max_=max_tree_depth)
self.toolbox.register(
"mutate", gp.mutUniform, expr=self.toolbox.expr_mut, pset=self.pset
)
self.toolbox.decorate(
"mate", gp.staticLimit(key=operator.attrgetter("height"), max_value=17)
)
self.toolbox.decorate(
"mutate", gp.staticLimit(key=operator.attrgetter("height"), max_value=17)
)
stats_fit = tools.Statistics(lambda ind: ind.fitness.values)
stats_size = tools.Statistics(len)
mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_size)
mstats.register("avg", np.mean)
mstats.register("std", np.std)
mstats.register("min", np.min)
mstats.register("max", np.max)
pop = self.toolbox.population(n=pop_size)
hof = tools.HallOfFame(1)
self.hof = hof
pop, log = algorithms.eaSimple(
pop,
self.toolbox,
crossover_rate,
mutation_rate,
n_gen,
stats=mstats,
halloffame=hof,
verbose=True,
)
self.best_heuristic = hof[0]
print("best_heuristic: ", self.best_heuristic)
self.func_heuristic = self.toolbox.compile(expr=self.best_heuristic)
self.policy = GPHHPolicy(
self.domain,
self.domain_model,
self.func_heuristic,
features=self.list_feature,
params_gphh=self.params_gphh,
recompute_cpm=True,
)
def _get_next_action(
self, observation: D.T_agent[D.T_observation]
) -> D.T_agent[D.T_concurrency[D.T_event]]:
action = self.policy.sample_action(observation)
# print('action_1: ', action.action)
return action
def _is_policy_defined_for(self, observation: D.T_agent[D.T_observation]) -> bool:
return True
def init_primitives(self, pset) -> PrimitiveSet:
for i in range(len(self.list_feature)):
pset.renameArguments(**{"ARG" + str(i): self.list_feature[i].value})
return pset
def evaluate_heuristic(self, individual, domains) -> float:
vals = []
func_heuristic = self.toolbox.compile(expr=individual)
# print('individual', individual)
for domain in domains:
###
initial_state = domain.get_initial_state()
do_model = build_do_domain(domain)
modes = [
initial_state.tasks_mode.get(j, 1)
for j in sorted(domain.get_tasks_ids())
]
modes = modes[1:-1]
cpm = self.cpm_data[domain]["cpm"]
cpm_esd = self.cpm_data[domain]["cpm_esd"]
raw_values = []
for task_id in domain.get_available_tasks(initial_state):
input_features = [
feature_function_map[lf](
domain=domain,
cpm=cpm,
cpm_esd=cpm_esd,
task_id=task_id,
state=initial_state,
)
for lf in self.list_feature
]
output_value = func_heuristic(*input_features)
raw_values.append(output_value)
normalized_values = [
x + 1
for x in sorted(
range(len(raw_values)), key=lambda k: raw_values[k], reverse=False
)
]
normalized_values_for_do = [
normalized_values[i] - 2
for i in range(len(normalized_values))
if normalized_values[i] not in {1, len(normalized_values)}
]
solution = RCPSPSolution(
problem=do_model,
rcpsp_permutation=normalized_values_for_do,
rcpsp_modes=modes,
)
last_activity = max(list(solution.rcpsp_schedule.keys()))
do_makespan = solution.rcpsp_schedule[last_activity]["end_time"]
vals.append(do_makespan)
fitness = [np.mean(vals)]
# fitness = [np.max(vals)]
return fitness
# def evaluate_heuristic_sgs_deviation(self, individual, domains) -> float:
# vals = []
# func_heuristic = self.toolbox.compile(expr=individual)
# # selected_domains = random.sample(domains, 3)
# selected_domains = domains
#
# for domain in selected_domains:
# policy = GPHHPolicy(domain, domain,
# func_heuristic,
# features=self.list_feature,
# params_gphh=self.params_gphh, recompute_cpm=False, cpm_data=self.cpm_data
# )
# state = domain.get_initial_state().copy()
# domain.set_inplace_environment(True) # we can use True because we don't use the value
#
# states, actions, values = rollout_episode(domain=domain,
# max_steps=10000,
# solver=policy,
# from_memory=state,
# verbose=False,
# outcome_formatter=lambda
# o: f'{o.observation} - cost: {o.value.cost:.2f}')
#
# makespan = states[-1].t
# ref_makespan = self.reference_makespans[domain]
# makespan_deviation = (makespan - ref_makespan) / ref_makespan
# # print('mk: ', makespan, ' - mk_dev: ', makespan_deviation, ' - ref: ', ref_makespan)
# vals.append(makespan_deviation)
#
# # fitness = [np.mean(vals)]
# fitness = [np.mean(vals)]
# return fitness
def initialize_cpm_data_for_training(self):
self.cpm_data = {}
for domain in self.training_domains:
do_model = build_do_domain(domain)
cpm, cpm_esd = compute_cpm(do_model)
self.cpm_data[domain] = {"cpm": cpm, "cpm_esd": cpm_esd}
def evaluate_heuristic_permutation(self, individual, domains) -> float:
vals = []
func_heuristic = self.toolbox.compile(expr=individual)
# print('individual', individual)
for domain in domains:
raw_values = []
initial_state = domain.get_initial_state()
regenerate_cpm = False
if regenerate_cpm:
do_model = build_do_domain(domain)
cpm, cpm_esd = compute_cpm(do_model)
else:
cpm = self.cpm_data[domain]["cpm"]
cpm_esd = self.cpm_data[domain]["cpm_esd"]
for task_id in domain.get_available_tasks(state=initial_state):
input_features = [
feature_function_map[lf](
domain=domain,
cpm=cpm,
cpm_esd=cpm_esd,
task_id=task_id,
state=initial_state,
)
for lf in self.list_feature
]
output_value = func_heuristic(*input_features)
raw_values.append(output_value)
most_common_raw_val = max(raw_values, key=raw_values.count)
most_common_count = raw_values.count(most_common_raw_val)
heuristic_permutation = [
x + 1
for x in sorted(
range(len(raw_values)), key=lambda k: raw_values[k], reverse=False
)
]
if self.permutation_distance == PermutationDistance.KTD:
dist, p_value = stats.kendalltau(
heuristic_permutation, self.reference_permutations[domain]
)
dist = -dist
if self.permutation_distance == PermutationDistance.HAMMING:
dist = distance.hamming(
heuristic_permutation, self.reference_permutations[domain]
)
if self.permutation_distance == PermutationDistance.KTD_HAMMING:
ktd, _ = stats.kendalltau(
heuristic_permutation, self.reference_permutations[domain]
)
dist = -ktd + distance.hamming(
heuristic_permutation, self.reference_permutations[domain]
)
penalty = most_common_count / len(raw_values)
# penalty = 0.
penalized_distance = dist + penalty
vals.append(penalized_distance)
fitness = [np.mean(vals)]
# fitness = [np.max(vals)]
return fitness
def test_features(self, domain, task_id, observation):
for f in FeatureEnum:
print("feature: ", f)
calculated_feature = feature_function_map[f](
domain=domain, task_id=task_id, state=observation
)
print("\tcalculated_feature: ", calculated_feature)
def set_domain(self, domain):
self.domain = domain
if self.policy is not None:
self.policy.domain = domain
#creator.create("Individual", np.ndarray, fitness=creator.Fitness)
# -
# Creation of and operation on individuals is conducted within the `Toolbox`.
#
# > A toolbox for evolution that contains the evolutionary operators. At first the toolbox contains a `clone()` method that duplicates any element it is passed as argument, this method defaults to the `copy.deepcopy()` function. and a `map()` method that applies the function given as first argument to every items of the iterables given as next arguments, this method defaults to the `map()` function. You may populate the toolbox with any other function by using the `register()` method.
#
#
# [DEAP documentation](https://deap.readthedocs.io/en/master/api/base.html?highlight=toolbox#toolbox)
toolbox = Toolbox()
# +
# item_id : package (as in: an item in a knapsack) - (instead of attr_item)
toolbox.register("item_id", random.randrange, n_packages)
print(f"toolbox.item_id selects package id,\ne.g, {toolbox.item_id()}")
# +
# individual: knapsack
toolbox.register("individual", initRepeat, creator.Individual,
toolbox.item_id, n_packages_per_knapsack)
print(f"toolbox.individual generates a knapsack from package_ids,\ne.g, {toolbox.individual()}")
# +
# population: population of knapsacks
def __call__(self):
_wf = wf(self.wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = ModelTimeEstimator(bandwidth=10)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
fixed_schedule = empty_fixed_schedule_part
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate, ga_functions=ga_functions,
fixed_schedule_part=fixed_schedule,
current_time=0.0, init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**self.GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0)
ga_makespan = Utility.makespan(resulted_schedule)
return (ga_makespan, logbook)
def register_crossover_operator(self, toolbox: Toolbox) -> None:
evlutiz_crossover = EvolutizCrossover(self.evolutiz_connector)
toolbox.register("mate", evlutiz_crossover.crossover)
def mainPart(x_,
y_,
pset,
max_=5,
pop_n=100,
random_seed=2,
cxpb=0.8,
mutpb=0.1,
ngen=5,
tournsize=3,
max_value=10,
double=False,
score=None,
cal_dim=True,
target_dim=None,
inter_add=True,
iner_add=True,
random_add=False,
store=True):
"""
Parameters
----------
target_dim
max_
inter_add
iner_add
random_add
cal_dim
score
double
x_
y_
pset
pop_n
random_seed
cxpb
mutpb
ngen
tournsize
max_value
Returns
-------
"""
if score is None:
score = [r2_score, explained_variance_score]
if cal_dim:
assert all([isinstance(i, Dim) for i in pset.dim_list
]), "all import dim of pset should be Dim object"
random.seed(random_seed)
toolbox = Toolbox()
if isinstance(pset, ExpressionSet):
PTrees = ExpressionTree
Generate = genHalfAndHalf
mutate = mutNodeReplacement
mate = cxOnePoint
elif isinstance(pset, FixedSet):
PTrees = FixedTree
Generate = generate_index
mutate = mutUniForm_index
mate = partial(cxOnePoint_index, pset=pset)
else:
raise NotImplementedError("get wrong pset")
if double:
Fitness_ = creator.create("Fitness_", Fitness, weights=(1.0, 1.0))
else:
Fitness_ = creator.create("Fitness_", Fitness, weights=(1.0, ))
PTrees_ = creator.create("PTrees_",
PTrees,
fitness=Fitness_,
dim=dnan,
withdim=0)
toolbox.register("generate", Generate, pset=pset, min_=1, max_=max_)
toolbox.register("individual",
initIterate,
container=PTrees_,
generator=toolbox.generate)
toolbox.register('population',
initRepeat,
container=list,
func=toolbox.individual)
# def selection
toolbox.register("select_gs", selTournament, tournsize=tournsize)
toolbox.register("select_kbest_target_dim",
selKbestDim,
dim_type=target_dim,
fuzzy=True)
toolbox.register("select_kbest_dimless", selKbestDim, dim_type="integer")
toolbox.register("select_kbest", selKbestDim, dim_type='ignore')
# def mate
toolbox.register("mate", mate)
# def mutate
toolbox.register("mutate", mutate, pset=pset)
if isinstance(pset, ExpressionSet):
toolbox.decorate(
"mate",
staticLimit(key=operator.attrgetter("height"),
max_value=max_value))
toolbox.decorate(
"mutate",
staticLimit(key=operator.attrgetter("height"),
max_value=max_value))
# def elaluate
toolbox.register("evaluate",
calculatePrecision,
pset=pset,
x=x_,
y=y_,
scoring=score[0],
cal_dim=cal_dim,
inter_add=inter_add,
iner_add=iner_add,
random_add=random_add)
toolbox.register("evaluate2",
calculatePrecision,
pset=pset,
x=x_,
y=y_,
scoring=score[1],
cal_dim=cal_dim,
inter_add=inter_add,
iner_add=iner_add,
random_add=random_add)
toolbox.register("parallel",
parallelize,
n_jobs=1,
func=toolbox.evaluate,
respective=False)
toolbox.register("parallel2",
parallelize,
n_jobs=1,
func=toolbox.evaluate2,
respective=False)
pop = toolbox.population(n=pop_n)
haln = 10
hof = HallOfFame(haln)
stats1 = Statistics(lambda ind: ind.fitness.values[0]
if ind and ind.y_dim in target_dim else 0)
stats1.register("max", np.max)
stats2 = Statistics(lambda ind: ind.y_dim in target_dim if ind else 0)
stats2.register("countable_number", np.sum)
stats = MultiStatistics(score1=stats1, score2=stats2)
population, logbook = eaSimple(pop,
toolbox,
cxpb=cxpb,
mutpb=mutpb,
ngen=ngen,
stats=stats,
halloffame=hof,
pset=pset,
store=store)
return population, hof
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule).entity
initial_state = State()
initial_state.mapping = heft_mapping
# initial_state.mapping = generate(_wf, rm, estimator, 1)[0].entity
initial_state.ordering = sorted_tasks
T, N = 20, 1000
toolbox = Toolbox()
toolbox.register("energy", energy, _wf, rm, estimator)
toolbox.register("update_T", update_T, T)
toolbox.register("neighbor", mapping_neighbor, _wf, rm, estimator, 1)
toolbox.register("transition_probability", transition_probability)
# use just a const to define number of attempts
toolbox.register("attempts_count", lambda T: 100)
logbook = tools.Logbook()
logbook.header = ["gen", "T", "val"]
stats = tools.Statistics(lambda ind: ind.energy.values[0])
stats.register("val", lambda arr: arr[0])
def do_exp():
best, log, current = run_sa(
toolbox=toolbox,