def check_solution(self): if self.schedule is None: return # Verify that a schedule object in a valid state self.schedule.verify(only_counters=not self.scheduler.rotations) failed = False # Check whether all jobs finish fjobs = Job.from_schedule(self.schedule, self.jobs) for sj, fj in zip(self.jobs, fjobs): if fj.completed: continue log.error( "Job {} is not completed at the end of the schedule: ".format( sj.to_str(), fj.to_str())) failed = True # Check all jobs meet deadlines for j, js in self.schedule.per_requests().items(): if j.deadline >= js[-1].end_time: continue log.error("Job {} does not meet deadline, finished={:.3f}".format( j.to_str(), js[-1].end_time)) failed = True if failed: log.error("The error occured in the generated schedule:\n") for s in self.schedule: log.error("{}".format(s.to_str())) sys.exit(1)
def _generate_schedule_jointly(self): """Generate schedule for multiple requests jointly.""" # Schedule all jobs in once new_requests = [ r for r in self.requests.keys() if r.status == JobRequestStatus.NEW ] schedule = self._generate_schedule(new_requests, allow_partial_solution=True) if not schedule: log.debug(f"Schedule was not generated") for request in new_requests: log.debug(f"Request {request.app.name} is rejected") request.status = JobRequestStatus.REFUSED self._remove_request(request) return None scheduled_requests = schedule.get_requests() for request in list(self.requests.keys()): if request.status == JobRequestStatus.ACCEPTED: if request not in scheduled_requests: raise RuntimeError( f"The earlier accepted request {request.app.name} " "was not scheduled") continue assert request.status == JobRequestStatus.NEW if request in scheduled_requests: log.debug(f"Request {request.app.name} is accepted") request.status = JobRequestStatus.ACCEPTED job = Job.from_request(request).dispatch() self.requests.update({request: job}) else: log.debug(f"Request {request.app.name} is rejected") request.status = JobRequestStatus.REFUSED self._remove_request(request) return schedule
def _generate_schedule(self, new_requests=None, allow_partial_solution=False): """Generate a schedule with new requests. Internal method to generate a schedule. The schedule is not applied in the resource manager. """ # Create a copy of the current job list with the new request # Ensure that jobs are immutable jobs = [j for _, j in self.requests.items() if j] if new_requests: for request in new_requests: jobs.append(Job.from_request(request).dispatch()) # Generate scheduling with the new job st = time.time() if allow_partial_solution: schedule = self.scheduler.schedule( jobs, scheduling_start_time=self.state_time, allow_partial_solution=True, current_schedule=self.schedule, ) else: schedule = self.scheduler.schedule( jobs, scheduling_start_time=self.state_time, ) et = time.time() log.debug(f"Schedule found = {schedule is not None}. " f"Time to find the schedule: {et-st}") return schedule
def __form_segment_variants(self, init_mappings): """Construct schedule segment variants given the mappings. Args: mappings (list of Mapping): The list of job mappings. Returns: a list of pairs (Schedule, list of Jobs) objects """ # 1. Check whether we need to find all variants. # This is determined, by whether there is any idle jobs with already # specified mapping. if not self.scheduler.migrations: all_variants = any( m is None and j.last_mapping is not None for j, m in zip(self.__jobs, init_mappings) ) else: all_variants = False # 2. Rotate mappings if self.scheduler.rotations: mappings_variants = self._rotate_mappings( init_mappings, all_variants=all_variants ) else: mappings_variants = [init_mappings] for mappings in mappings_variants: # 3. Construct the schedule segment segment = self.__form_schedule_segment(mappings) if segment is None: continue # 4. Check that all jobs meet dealines if any(js.end_time > js.request.deadline for js in segment.jobs()): continue # Generate the job states at the end of the segment njobs = [ x for x in Job.from_schedule( Schedule(self.platform, [segment]), self.__jobs ) if not x.is_terminated() ] # 5. Check whether all remaining jobs still meet deadlines if not all(j.can_meet_deadline(segment.end_time) for j in njobs): continue # 6. Save segment new_schedule = self.__prev_schedule.copy() new_schedule.add_segment(segment) self.__results.append((new_schedule, njobs))
def __init__(self, scheduler, reqs): self.scheduler = scheduler self.requests = reqs # Job table # TODO: no need in a method which generates the whole list, switch to # a single object constructor self.jobs = list( map(lambda x: x.dispatch(), Job.from_requests(self.requests))) log.info("Jobs: {}".format(",".join(x.to_str() for x in self.jobs))) # Scheduling results self.found_schedule = None self.schedule = None
def test_scheduler(platform, graph, pareto_mappings): request = JobRequestInfo(graph, pareto_mappings, arrival=0.0, deadline=10.0) job = Job.from_request(request) scheduler = MedfScheduler(platform) schedule = scheduler.schedule([job]) assert len(schedule.segments()) == 1 segment = schedule.segments()[0] assert segment.start_time == 0.0 assert segment.end_time == 9.7 assert len(segment.jobs()) == 1 assert segment.energy == 23.45
def _generate_schedule_iteratively(self): """Generate schedule for multiple requests iteratively.""" new_requests = [ r for r in self.requests.keys() if r.status == JobRequestStatus.NEW ] new_schedule = None for request in new_requests: schedule = self._generate_schedule([request]) if schedule: log.debug(f"Request {request.app.name} is accepted") request.status = JobRequestStatus.ACCEPTED job = Job.from_request(request).dispatch() self.requests.update({request: job}) new_schedule = schedule else: log.debug(f"Request {request.app.name} is rejected") request.status = JobRequestStatus.REFUSED self._remove_request(request) return new_schedule
def _advance_segment(self, segment, till_time=None): """Advance an internal state by the schedule segment. This method advances the internal state by a single segment. If `till_time` is not None, then the segment is advanced till this specified time within a segment. If `till_time` is None, the function returns None, otherwise it returns the remaining part of the segment """ # the state time must equal to start time of the segment if abs(self.state_time - segment.start_time) > 0.0001: raise RuntimeError( f"The current state time ({self.state_time}) does not equal " f"the start time of the segment ({segment.start_time})") rest = None if till_time: if not segment.start_time <= till_time <= segment.end_time: raise RuntimeError( f"The end time ({till_time}) must be within the segment " f"({segment.start_time}..{segment.end_time})") segment, rest = segment.split(till_time) jobs = [j for _, j in self.requests.items() if j] schedule = Schedule(self.platform, [segment]) # FIXME: Remove this constructor, create method in # SingleJobMappingSegment class end_jobs = Job.from_schedule(schedule, init_jobs=jobs) for job in end_jobs: request = job.request if job.is_terminated(): self._finish_request(request) else: self.requests.update({request: job}) self._state_time = segment.end_time self._dynamic_energy += segment.energy return rest
def generate_segment(self, jobs, segment_start_time=0.0): # Solve Lagrangian relaxation of MMKP log.debug("Solving Lagrangian relaxation of MMKP...") l, job_mappings = self.__lr_solver.solve( jobs, segment_start_time=segment_start_time ) log.debug("Found lambda = {}".format(l)) for j, m in job_mappings: log.debug( "Job {}, mapping {} [e:{:.3f}], f = {:.3f}".format( j.to_str(), m.get_used_processor_types(), m.metadata.energy, LRSolver.job_config_cost(j, m, l), ) ) # Sort applications by a defined sorting key if self.__sorting_key == SegLRSortingKey.MINCOST: # Sort applications that f(x_i*, lambda*) <= f(x_j*, lambda*), i < j job_mappings.sort( key=lambda x: (LRSolver.job_config_cost(x[0], x[1], l)) ) elif self.__sorting_key == SegLRSortingKey.DEADLINE: assert False, "NYI" min_configs.sort(key=lambda j: j[0].deadline) elif self.__sorting_key == SegLR15SortingKey.MINCOST_DEADLINE_PRODUCT: assert False, "NYI" min_configs.sort( key=lambda j: ( LRSolver.job_config_costf(j[0], j[2], l) * j[0].deadline ) ) else: assert False, "Sorting key is not known: {}".format( self.__sorting_key ) # Empty resource avail_cores = self.platform.get_processor_types() # Empty segment mapping final_job_mappings = {} min_rtime = math.inf # Map incrementally jobs for job, _ in job_mappings: log.debug("Selecting a mapping for {}".format(job.to_str())) log.debug("Available resources: {}".format(avail_cores)) cratio = job.cratio rratio = 1.0 - cratio # Get all configurations of the job, sort them by f(x_i, lambda) cost_mappings = [ (LRSolver.job_config_cost(job, m, l), m) for m in job.request.mappings ] cost_mappings.sort(key=lambda x: x[0]) assert self.__explore_mode == SegLRExploreMode.ALL, "NYI" added = False for cost, mapping in cost_mappings: # Check if the current mapping fits resources map_cores = mapping.get_used_processor_types() log.debug( "... Checking mapping: {}, t*:{:.3f}, e*:{:.3f}, f:{:.3f}".format( map_cores, mapping.metadata.exec_time * rratio, mapping.metadata.energy * rratio, cost, ) ) if map_cores | avail_cores != avail_cores: log.debug("....... Not enough resources") continue # It is possible to map on the available resources, check # whether it satisfies its deadline condition. if ( mapping.metadata.exec_time * rratio + segment_start_time > job.deadline ): # This mapping cannot fit deadline condition. # TODO: test whether it is useful if self.__allow_local_violations: # Check whether we may compensate the rate of job # at the end of the segment if min_rtime + segment_start_time > job.deadline: log.debug("....... Cannot meet deadline") continue # Construct a temporary job_segment job_segment = SingleJobSegmentMapping( job.request, mapping, start_time=segment_start_time, start_cratio=cratio, end_time=segment_start_time + min_rtime, ) end_cratio = job_segment.end_cratio bctime = min( [m.metadata.exec_time for m in job.request.mappings] ) * (1.0 - end_cratio) if ( min_rtime + bctime + segment_start_time > job.deadline ): # This configuration cannot be compensated log.debug( "....... Cannot meet deadline at the end of the current segment after selecting this mapping" ) continue final_job_mappings[job] = mapping log.debug("....... Selected") else: assert False, "NYI" log.debug("....... Cannot meet deadline") continue else: # This configuration can fit the deadline final_job_mappings[job] = mapping rtime = mapping.metadata.exec_time * rratio min_rtime = min(min_rtime, rtime) log.debug("....... Selected") avail_cores -= map_cores added = True break if all(m is None for m in final_job_mappings): return None # Calculate segment end time jobs_rem_time = [ m.metadata.exec_time * (1.0 - j.cratio) for j, m in final_job_mappings.items() if m is not None ] # TODO: Allow MAX_END_GAP at the end of the segment # * Make sure that min_rtime also includes such MAX_END_GAP # segment_duration = max( # [t for t in jobs_rem_time if t < min(jobs_rem_time) + MAX_END_GAP]) segment_duration = min(jobs_rem_time) segment_end_time = segment_duration + segment_start_time # Construct the job_mappings job_segments = [] for j, m in final_job_mappings.items(): if m is None: assert False, "NYI" continue ssm = SingleJobSegmentMapping( j.request, m, start_time=segment_start_time, start_cratio=j.cratio, end_time=segment_end_time, ) job_segments.append(ssm) # Construct a schedule segment new_segment = MultiJobSegmentMapping(self.platform, job_segments) new_segment.verify(only_counters=not self.scheduler.rotations) # Check that idle jobs do not miss deadline for j, m in final_job_mappings.items(): if m is not None: continue if j.deadline < shortest_end_time: assert False, "NYI" return None # Generate the job states at the end of the segment new_jobs = [ x for x in Job.from_schedule( Schedule(self.platform, [new_segment]), jobs ) if not x.is_terminated() ] log.debug("Generated segment: {}".format(new_segment.to_str())) return new_segment, new_jobs
def generate_segment(self, jobs, segment_start_time=0.0): self.__segment_start_time = segment_start_time job_mappings = {} avl_core_types = self.platform.get_processor_types() while any(j not in job_mappings for j in jobs): log.debug("Free cores: {}".format(avl_core_types)) # List of mappings to finish the applications to_finish = {} diff = (-math.inf, None) for job in jobs: # Skip jobs with found mappings if job in job_mappings: continue to_finish[ job] = self._filter_job_mappings_by_deadline_resources( job, avl_core_types) to_finish[job].sort(key=lambda m: m.metadata.energy) log.debug("to_finish[{}]: {}".format( job.to_str(), [ m.metadata.energy * (1.0 - job.cratio) for m in to_finish[job] ], )) if len(to_finish[job]) == 0: job_mappings[job] = None continue if len(to_finish[job]) == 1: diff = (math.inf, job) continue # Check energy difference between first two mappings cdiff = (1.0 - job.cratio) * (to_finish[job][1].metadata.energy - to_finish[job][0].metadata.energy) if cdiff > diff[0]: diff = (cdiff, job) _, job_d = diff if job_d is None: return None log.debug("Choose {}".format(job_d.to_str())) # On Odroid XU-4, the check that it can be scheduled is simple. # All mappings in to_finish are valid. m = to_finish[job_d][0] avl_core_types -= m.get_used_processor_types() job_mappings[job_d] = m log.debug("Job_mappings: {}".format([( j.to_str(), m.get_used_processor_types(), m.metadata.energy * (1.0 - j.cratio), ) if m is not None else (j.to_str(), None) for j, m in job_mappings.items()])) segment = self._form_schedule_segment(job_mappings) # TODO: this is copied from bruteforce, put it in a separate functions if segment is None: return None # Check that all jobs meet dealines for js in segment.jobs(): if js.end_time > js.request.deadline: return None # Generate the job states at the end of the segment new_jobs = [ x for x in Job.from_schedule(Schedule(self.platform, [segment]), jobs) if not x.is_terminated() ] # Check whether all remaining jobs still meet deadlines if not all(j.can_meet_deadline(segment.end_time) for j in new_jobs): return None return segment, new_jobs