def __init__(self, workflow, resource_manager, estimator):
self.counter = 0
self.workflow = workflow
##interface Estimator
self.estimator = estimator
self.resource_manager = resource_manager
nodes = resource_manager.get_nodes(
) #list(HeftHelper.to_nodes(resource_manager.get_resources()))
ranking = HeftHelper.build_ranking_func(
nodes, lambda job, agent: estimator.estimate_runtime(job, agent),
lambda ni, nj, A, B: estimator.estimate_transfer_time(
A, B, ni, nj))
sorted_tasks = ranking(self.workflow)
self.nodes = nodes
self.sorted_tasks = sorted_tasks
self.workflow_size = len(sorted_tasks)
self.task_map = {task.id: task for task in sorted_tasks}
self.node_map = {node.name: node for node in nodes}
self.initializing_alg = SimpleRandomizedHeuristic(
self.workflow, self.nodes, self.estimator)
self.initial_chromosome = None ##GAFunctions.schedule_to_chromosome(initial_schedule)
pass
def generate(wf, rm, estimator, n):
pop = []
for i in range(n):
sched = SimpleRandomizedHeuristic(wf, rm.get_nodes(), estimator).schedule()
particle = schedule_to_position(sched)
particle.velocity = MappingParticle.Velocity({})
pop.append(particle)
return pop
def generate(n):
schedules = [SimpleRandomizedHeuristic(_wf, rm.get_nodes(), estimator).schedule() for _ in range(n)]
mapping_positions = [schedule_to_position(s).entity for s in schedules]
ordering_individuals = [ordering_from_schedule(s) for s in schedules]
pop = []
for mp, os in zip(mapping_positions, ordering_individuals):
p = ParticleIndividual(Position(mp))
p.ordering = os
pop.append(p)
return pop
def generate(wf, rm, estimator, schedule=None, fixed_schedule_part=None, current_time=0.0):
sched = schedule if schedule is not None else SimpleRandomizedHeuristic(wf, rm.get_nodes(), estimator).schedule(fixed_schedule_part, current_time)
if fixed_schedule_part is not None:
un_tasks = unmoveable_tasks(fixed_schedule_part)
clean_sched = Schedule({node: [item for item in items if item.job.id not in un_tasks and item.state != ScheduleItem.FAILED]
for node, items in sched.mapping.items()})
else:
clean_sched = sched
mapping, ordering = ord_and_map(clean_sched)
ordering_numseq = ordering_to_numseq(ordering)
ordering_map = {task_id: val for task_id, val in zip(ordering, ordering_numseq)}
ord_p, map_p = OrderingParticle(ordering_map), MappingParticle(mapping)
ord_p.velocity = OrderingParticle.Velocity({})
map_p.velocity = MappingParticle.Velocity({})
result = CompoundParticle(map_p, ord_p)
if schedule is None and not validate_mapping_with_alive_nodes(result.mapping.entity, rm):
raise Exception("found invalid solution in generated array")
return result
def generate(wf, rm, estimator):
sched = SimpleRandomizedHeuristic(wf, rm.get_nodes(), estimator).schedule()
return schedule_to_position(sched)
class GAFunctions2:
## A chromosome representation
##{
## node_name: task1.id, task2.id, ... #(order of tasks is important)
## ...
##}
def __init__(self, workflow, resource_manager, estimator):
self.counter = 0
self.workflow = workflow
##interface Estimator
self.estimator = estimator
self.resource_manager = resource_manager
nodes = resource_manager.get_nodes(
) #list(HeftHelper.to_nodes(resource_manager.get_resources()))
ranking = HeftHelper.build_ranking_func(
nodes, lambda job, agent: estimator.estimate_runtime(job, agent),
lambda ni, nj, A, B: estimator.estimate_transfer_time(
A, B, ni, nj))
sorted_tasks = ranking(self.workflow)
self.nodes = nodes
self.sorted_tasks = sorted_tasks
self.workflow_size = len(sorted_tasks)
self.task_map = {task.id: task for task in sorted_tasks}
self.node_map = {node.name: node for node in nodes}
self.initializing_alg = SimpleRandomizedHeuristic(
self.workflow, self.nodes, self.estimator)
self.initial_chromosome = None ##GAFunctions.schedule_to_chromosome(initial_schedule)
pass
@staticmethod
def schedule_to_chromosome(schedule, fixed_schedule_part=None):
if schedule is None:
return None
def ids(items):
return [
item.job.id for item in items
if item.state != ScheduleItem.FAILED
]
chromosome = {
node.name: ids(items)
for (node, items) in schedule.mapping.items()
}
if fixed_schedule_part is None:
return chromosome
fixed_chromosome = {
node.name: ids(items)
for (node, items) in fixed_schedule_part.mapping.items()
}
for node, tasks in fixed_chromosome.items():
for t in tasks:
chromosome[node].remove(t)
count = 0
for node, tasks in chromosome.items():
count += len(tasks)
if count == 0:
return None
return chromosome
def build_initial(self, fixed_schedule_part, current_time):
def initial():
return self.random_chromo(fixed_schedule_part, current_time)
return initial
def random_chromo(self, fixed_schedule_part, current_time):
res = random.random()
# # # TODO:
if res > 1.00 and self.initial_chromosome is not None:
return self.initial_chromosome
##return [self.random_chromo() for j in range(self.size)]
sched = self.initializing_alg.schedule(fixed_schedule_part,
current_time)
#TODO: remove it later
# mark_finished(sched)
# seq_time_validaty = Utility.validateNodesSeq(sched)
# dependency_validaty = Utility.validateParentsAndChildren(sched, self.workflow)
## TODO: Urgent! Need to remove failed Tasks
chromo = GAFunctions2.schedule_to_chromosome(sched)
if fixed_schedule_part is not None:
# remove fixed_schedule_part from chromosome
# def is_last_version_of_task_executing(item):
# return item.state == ScheduleItem.EXECUTING or item.state == ScheduleItem.FINISHED or item.state == ScheduleItem.UNSTARTED
# # there is only finished, executing and unstarted tasks planned by Heft, perhaps it should have been called unplanned tasks
# finished_tasks = [item.job.id for (node, items) in fixed_schedule_part.mapping.items() for item in items if is_last_version_of_task_executing(item)]
finished_tasks = unmoveable_tasks(fixed_schedule_part)
# TODO: make common utility function with ScheduleBuilder and SimpleRandomizedHeuristic
chromo = {
node_name: [id for id in ids if not (id in finished_tasks)]
for (node_name, ids) in chromo.items()
}
return chromo
def build_fitness(self, fixed_schedule_part, current_time):
# builder = ScheduleBuilder(self.workflow, self.resource_manager, self.estimator, self.task_map, self.node_map, fixed_schedule_part)
builder = NewScheduleBuilder(self.workflow, self.resource_manager,
self.estimator, self.task_map,
self.node_map, fixed_schedule_part)
def fitness(chromo):
## TODO: remove it later.
# t_ident = str(threading.current_thread().ident)
# t_name = str(threading.current_thread().name)
# print("Time: " + str(current_time) + " Running ga in isolated thread " + t_name + " " + t_ident)
## value of fitness function is the last time point in the schedule
## built from the chromo
## chromo is {Task:Node},{Task:Node},... - fixed length
schedule = builder(chromo, current_time)
time = Utility.makespan(schedule)
# time = 1
# ## TODO: remove it later.
# k = 0
# for i in range(100000):
# k += i
return (1 / time, )
## TODO: redesign it later
return fitness
def build_schedule(self, chromo, fixed_schedule_part, current_time):
# builder = ScheduleBuilder(self.workflow, self.resource_manager, self.estimator, self.task_map, self.node_map, fixed_schedule_part)
builder = NewScheduleBuilder(self.workflow, self.resource_manager,
self.estimator, self.task_map,
self.node_map, fixed_schedule_part)
schedule = builder(chromo, current_time)
return schedule
def crossover(self, child1, child2):
## TODO: only for debug. remove it later.
##return
#estimate size of a chromosome
alive_nodes = [node for node in self.nodes if node.state != Node.Down]
if len(alive_nodes) == 0:
raise Exception(" There are only dead nodes!!!!!!!!!!!!!")
# TODO: corner case when fixed schedule is complete. need to resolve this kind of case early.
size = len(
[item for (node_name, items) in child1.items() for item in items])
if size == 0:
raise Exception("Chromosome is empty")
# return None
i1 = random.randint(0, size - 1)
i2 = random.randint(0, size - 1)
index1 = min(i1, i2)
index2 = max(i1, i2)
def chromo_to_seq(chromo):
result = []
for (nd_name, items) in chromo.items():
result += [(nd_name, item) for item in items]
return result
def fill_chromo(chromo, seq):
chromo.clear()
for node in self.nodes:
chromo[node.name] = []
for (nd_name, tsk_id) in seq:
chromo[nd_name].append(tsk_id)
ch1 = chromo_to_seq(child1)
ch2 = chromo_to_seq(child2)
if len(ch1) != size or len(ch2) != size:
print("Ch1: {0}".format(len(ch1)))
print("Ch2: {0}".format(len(ch2)))
raise Exception("Transformed chromosome is broken")
window = dict()
for i in range(index1, index2):
tsk_id = ch1[i][1]
window[ch1[i][1]] = i
for i in range(size):
tsk_id = ch2[i][1]
if tsk_id in window:
buf = ch1[window[tsk_id]]
ch1[window[tsk_id]] = ch2[i]
ch2[i] = buf
fill_chromo(child1, ch1)
fill_chromo(child2, ch2)
return child1, child2
# def swap_mutation(self, chromo):
# node_index = random.randint(0, len(self.nodes) - 1)
# node_seq = chromo[self.nodes[node_index].name]
#
# while True:
def mutation(self, chromosome):
## TODO: only for debug. remove it later.
#return chromosome
# simply change one node of task mapping
#TODO: make checking for all nodes are dead.(It's a very rare situation so it is not consider for now)
alive_nodes = [node for node in self.nodes if node.state != Node.Down]
node1 = alive_nodes[random.randint(0, len(alive_nodes) - 1)]
node2 = alive_nodes[random.randint(0, len(alive_nodes) - 1)]
ch = chromosome[node1.name]
if len(chromosome[node1.name]) > 0:
length = len(chromosome[node1.name])
ind = random.randint(0, length - 1)
dna = chromosome[node1.name][ind]
del chromosome[node1.name][ind]
chromosome[node2.name].append(dna)
return chromosome
def sweep_mutation(self, chromosome):
## TODO: only for debug. remove it later.
#return chromosome
def is_dependent(tsk1, tsk2):
for p in tsk1.parents:
if tsk1.id == tsk2.id:
return True
else:
return is_dependent(p, tsk2)
return False
#return chromosome
#TODO: make checking for all nodes are dead.(It's a very rare situation so it is not considered for now)
alive_nodes = [node for node in self.nodes if node.state != Node.Down]
node = alive_nodes[random.randint(0, len(alive_nodes) - 1)]
ch = chromosome[node.name]
if len(chromosome[node.name]) > 0:
length = len(chromosome[node.name])
ind = random.randint(0, length - 1)
tsk1 = self.task_map[chromosome[node.name][ind]]
dna = chromosome[node.name][ind]
count = 0
while count < 5:
ind1 = random.randint(0, length - 1)
tsk2 = self.task_map[chromosome[node.name][ind1]]
if (not is_dependent(tsk1, tsk2)) and (not is_dependent(
tsk2, tsk1)):
chromosome[node.name][ind] = chromosome[node.name][ind1]
chromosome[node.name][ind1] = dna
break
else:
count += 1
return chromosome
pass