def schedule(self): """ create inter-priority """ def byPriority(wf): return 0 if wf.priority is None else wf.priority ##simple inter priority sorting sorted_wfs = sorted(self.workflows, key=byPriority) wf_jobs = {wf: [] for wf in sorted_wfs} resources = self.resource_manager.get_resources() ##print("common nodes count:" + str(len(toNodes(resources)))) nodes = HeftHelper.to_nodes(resources) wf_jobs = {wf: self.make_ranking(wf, nodes) for wf in sorted_wfs} ##new_schedule = self.get_unchanged_schedule(self.old_schedule, time) new_schedule = Schedule({node: [] for node in nodes}) new_plan = new_schedule.mapping for (wf, jobs) in wf_jobs.items(): new_schedule = self.mapping([(wf, jobs)], new_plan, nodes, self.commcost, self.compcost) new_plan = new_schedule.mapping return new_schedule
def run(self, current_cleaned_schedule): ## current_cleaned_schedule - this schedule contains only ## finished and executed tasks, all unfinished and failed have been removed already ## current_cleaned_schedule also have down nodes and new added ## ALGORITHM DOESN'T CHECK ADDING OF NEW NODES BY ITSELF ## nodes contain only available now ## 1. get all unscheduled tasks ## 2. sort them by rank ## 3. map on the existed nodes according to current_cleaned_schedule nodes = self.get_nodes() live_nodes = [node for node in nodes if node.state != Node.Down] for_planning = HeftHelper.get_tasks_for_planning(self.workflow, current_cleaned_schedule) ## TODO: check if it sorted properly for_planning = set([task.id for task in for_planning]) self.wf_jobs = self.make_ranking(self.workflow, live_nodes) if self.ranking is None else self.ranking sorted_tasks = [task for task in self.wf_jobs if task.id in for_planning] # print("P: " + str(sorted_tasks)) new_sched = self.mapping([(self.workflow, sorted_tasks)], current_cleaned_schedule.mapping, nodes, self.commcost, self.compcost) return new_sched
def make_ranking(self, wf, nodes): ##resources = self.resource_manager.get_resources() ##print("common nodes count:" + str(len(toNodes(resources)))) ##nodes = HeftHelper.to_nodes(resources) ranking_func = HeftHelper.build_ranking_func(nodes, self.compcost, self.commcost) wf_jobs = ranking_func(wf) return wf_jobs
def mapping(self, sorted_jobs, existing_plan, live_nodes, commcost, compcost): """def allocate(job, orders, jobson, prec, compcost, commcost):""" """ Allocate job to the machine with earliest finish time Operates in place """ ## TODO: add finished tasks jobson = dict() for (node, items) in existing_plan.items(): for item in items: if item.state == ScheduleItem.FINISHED or item.state == ScheduleItem.EXECUTING: jobson[item.job] = node new_plan = existing_plan def ft(machine): #cost = st(machine) runtime = compcost(task, machine) cost = st(machine, runtime) + runtime ##print("machine: %s job:%s cost: %s" % (machine.name, task.id, cost)) ##print("machine: " + str(machine.name) + " cost: " + str(cost)) return cost if len(live_nodes) != 0: ## in case if there is not any live nodes we just return the same cleaned schedule for wf, tasks in sorted_jobs: ##wf_dag = self.convert_to_parent_children_map(wf) wf_dag = HeftHelper.convert_to_parent_children_map(wf) prec = reverse_dict(wf_dag) for task in tasks: st = partial(self.start_time, wf, task, new_plan, jobson, prec, commcost) # ress = [(key, ft(key)) for key in new_plan.keys()] # agent_pair = min(ress, key=lambda x: x[1][0]) # agent = agent_pair[0] # start = agent_pair[1][0] # end = agent_pair[1][1] # agent = min(new_plan.keys(), key=ft) agent = min(live_nodes, key=ft) runtime = compcost(task, agent) start = st(agent, runtime) end = ft(agent) # new_plan[agent].append(ScheduleItem(task, start, end)) Schedule.insert_item(new_plan, agent, ScheduleItem(task, start, end)) jobson[task] = agent new_sched = Schedule(new_plan) return new_sched
def checkDown(self, node_name, is_down): nodes = HeftHelper.to_nodes(self.public_resources) for nd in nodes: if nd.name == node_name: if is_down: nd.state = Node.Down else: nd.state = Node.Unknown pass
def run(self, current_cleaned_schedule): nodes = self.get_nodes() live_nodes = [node for node in nodes if node.state != Node.Down] for_planning = HeftHelper.get_tasks_for_planning(self.workflow, current_cleaned_schedule) for_planning = set([task.id for task in for_planning]) self.wf_jobs = self.make_ranking(self.workflow, live_nodes) if self.ranking is None else self.ranking sorted_tasks = [task for task in self.wf_jobs if task.id in for_planning] # print("P: " + str(sorted_tasks)) new_sched = self.mapping([(self.workflow, sorted_tasks)], current_cleaned_schedule.mapping, nodes, self.commcost, self.compcost) return new_sched
def __init__(self, workflow, resource_manager, estimator, task_map, node_map, # fixed part of schedule. It need to be accounted when new schedule is built, but it's not possible to cahnge something inside it fixed_schedule_part): self.workflow = workflow self.nodes = HeftHelper.to_nodes(resource_manager.get_resources()) self.estimator = estimator ##TODO: Build it self.task_map = task_map ##TODO: Build it self.node_map = node_map self.fixed_schedule_part = fixed_schedule_part # construct initial mapping # eliminate all already scheduled tasks pass
def isCloudNode(self, node): result = node.name in [nd.name for nd in HeftHelper.to_nodes(self.public_resources)] return result
def get_by_softreq(self, soft_reqs): nodes = HeftHelper.to_nodes(self.public_resources) def check_reqs(node): return (soft_reqs in node.soft) or (SoftItem.ANY_SOFT in node.soft) gotcha = [node for node in nodes if node.state != Node.Down and check_reqs(node)] return gotcha
def get_nodes(self): resources = self.resource_manager.get_resources() nodes = HeftHelper.to_nodes(resources) return nodes
def generate_public_resources(self): ## TODO: remake it later #(public_resources, generate_reliability, generate_probability_law_for_(task,node)_pair) = generate public_resource resCount = 3 resources = list() for i in range(0, resCount): res = Resource("public_res_" + str(i)) resources.append(res) nodeCount = None if i == 0: nodeCount = 15 elif i == 1: nodeCount = 12 elif i == 2: nodeCount = 9 for j in range(0, nodeCount): node = Node(res.name + "_node_" + str(j), res, [SoftItem.ANY_SOFT]) # if j == 0: # node.flops = 10 + 5 # if j == 1: # node.flops = 15 + 10#10*3 # if j == 2: # node.flops = 25 + 10#25*3 # if j == 3: # node.flops = 25 + 10#25*3 # if j == 4: # node.flops = 30 + 10#30*3 # if j == 5: # node.flops = 10 + 5 # if j == 6: # node.flops = 15 + 10#10*3 # if j == 7: # node.flops = 25 + 10#25*3 # if j == 8: # node.flops = 25 + 10#25*3 # if j == 9: # node.flops = 30 + 10#30*3 # if j == 10: # node.flops = 10 + 5 # if j == 11: # node.flops = 15 + 10#10*3 # if j == 12: # node.flops = 25 + 10#25*3 # if j == 13: # node.flops = 25 + 10#25*3 # if j == 14: # node.flops = 30 + 10#30*3 if j == 0: node.flops = 10 if j == 1: node.flops = 15#10*3 if j == 2: node.flops = 25#25*3 if j == 3: node.flops = 25#25*3 if j == 4: node.flops = 30#30*3 if j == 5: node.flops = 10 if j == 6: node.flops = 15#10*3 if j == 7: node.flops = 25#25*3 if j == 8: node.flops = 25#25*3 if j == 9: node.flops = 30#30*3 if j == 10: node.flops = 10 if j == 11: node.flops = 15#10*3 if j == 12: node.flops = 25#25*3 if j == 13: node.flops = 25#25*3 if j == 14: node.flops = 30#30*3 res.nodes.add(node) nodes = HeftHelper.to_nodes(resources) reliability_map = {node.name: 0.9 for node in nodes} def probability_estimator(dt, comp_estimation, transfer_estimation): M = comp_estimation + transfer_estimation sigma = 0.1 * M result = 0.5 *(1 + math.erf((dt - M)/sigma)) return result return (resources, reliability_map, probability_estimator)