def sub_slot_during_job(self, slot, job): slot.b = max(slot.b, job.start_time) slot.e = min(slot.e, job.start_time + job.walltime - 1) slot.itvs = sub_intervals(slot.itvs, job.res_set) if job.ts: if job.ts_user not in slot.ts_itvs: slot.ts_itvs[job.ts_user] = {} if job.ts_name not in slot.ts_itvs[job.ts_user]: slot.ts_itvs[job.ts_user][job.ts_name] = job.res_set[:] if job.ph == ALLOW: if job.ph_name in slot.ph_itvs: slot.ph_itvs[job.ph_name] = \ sub_intervals(slot.ph_itvs[job.ph_name], job.res_set) if job.ph == PLACEHOLDER: slot.ph_itvs[job.ph_name] = job.res_set[:] if hasattr(slot, 'quotas') and not ("container" in job.types): slot.quotas.update(job)
def extract_scheduled_jobs(result, resource_set, job_security_time, now): jids = [] jobs_lst = [] jobs = {} prev_jid = 0 roids = [] rid2jid = {} global job # (job, a, b, c) = req[0] if result: for x in result: j, moldable_id, start_time, walltime, r_id = x if j.id != prev_jid: if prev_jid != 0: job.res_set = unordered_ids2itvs(roids) jobs_lst.append(job) jids.append(job.id) jobs[job.id] = job roids = [] prev_jid = j.id job = j job.start_time = start_time job.walltime = walltime + job_security_time job.moldable_id = moldable_id job.ts = False job.ph = NO_PLACEHOLDER job.assign = False job.find = False if job.suspended == "YES": job.walltime += get_job_suspended_sum_duration(job.id, now) roid = resource_set.rid_i2o[r_id] roids.append(roid) rid2jid[roid] = j.id job.res_set = unordered_ids2itvs(roids) if job.state == "Suspended": job.res_set = sub_intervals( job.res_set, resource_set.suspendable_roid_itvs) jobs_lst.append(job) jids.append(job.id) jobs[job.id] = job get_jobs_types(jids, jobs) return (jobs, jobs_lst, jids, rid2jid)
def handle_waiting_reservation_jobs(queue_name, resource_set, job_security_time, current_time_sec): logger.debug("Queue " + queue_name + ": begin processing accepted Advance Reservations") ar_jobs = get_waiting_scheduled_AR_jobs(queue_name, resource_set, job_security_time, current_time_sec) for job in ar_jobs: moldable_id = job.moldable_id walltime = job.walltime # Test if AR job is expired and handle it if (current_time_sec > (job.start_time + walltime)): logger.warn("[" + str(job.id) + "] set job state to Error: avdance reservation expired and couldn't be started") set_job_state(job.id, 'Error') set_job_message(job.id, "Reservation expired and couldn't be started.") else: # Determine current available ressources avail_res = intersec(resource_set.roid_itvs, job.res_set) # Test if the AR job is waiting to be launched due to nodes' unavailabilities if (avail_res == []) and (job.start_time < current_time_sec): logger.warn("[%s] advance reservation is waiting because no resource is present" % str(job.id)) # Delay launching time set_gantt_job_start_time(moldable_id, current_time_sec + 1) elif (job.start_time < current_time_sec): if (job.start_time + reservation_waiting_timeout) > current_time_sec: if not equal_itvs(avail_res, job.res_set): # The expected ressources are not all available, # wait the specified timeout logger.warn("[" + str(job.id) + "] advance reservation is waiting because not all \ resources are available yet") set_gantt_job_start_time(moldable_id, current_time_sec + 1) else: # It's time to launch the AR job, remove missing ressources missing_resources_itvs = sub_intervals(job.res_set, avail_res) remove_gantt_resource_job(moldable_id, missing_resources_itvs, resource_set) logger.warn("[" + str(job.id) + "remove some resources assigned to this advance reservation, \ because there are not Alive") add_new_event('SCHEDULER_REDUCE_NB_RESSOURCES_FOR_ADVANCE_RESERVATION', job.id, "[MetaSched] Reduce the number of resources for the job " + str(job.id)) nb_res = itvs_size(job.res_set) - itvs_size(missing_resources_itvs) new_message = re.sub(r'R=\d+', 'R=' + str(nb_res), job.message) if new_message != job.message: set_job_message(job.id, new_message) logger.debug("Queue " + queue_name + ": end processing of reservations with missing resources")
def schedule_fifo_cycle(plt, queue="default", hierarchy_use=False): assigned_jobs = {} now = plt.get_time() logger.info("Begin scheduling....now: " + str(now) + ", queue: " + queue) # # Retrieve waiting jobs # waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue) if nb_waiting_jobs > 0: logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs)) for jid in waiting_jids: logger.debug("waiting_jid: " + str(jid)) # # Determine Global Resource Intervals # resource_set = plt.resource_set() res_itvs = deepcopy(resource_set.roid_itvs) # # Get additional waiting jobs' data # job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"]) plt.get_data_jobs(waiting_jobs, waiting_jids, resource_set, job_security_time) # # Remove resources used by running job # for job in plt.get_scheduled_jobs(resource_set, job_security_time, now): if job.state == "Running": res_itvs = sub_intervals(res_itvs, job.res_itvs) # # Assign resource to jobs # for jid in waiting_jids: job = waiting_jobs[jid] # We consider only one instance of resources request (no support for moldable) (mld_id, walltime, hy_res_rqts) = job.mld_res_rqts[0] if hierarchy_use: # Assign resources which hierarchy support (uncomment) itvs = find_resource_hierarchies_job(res_itvs, hy_res_rqts, resource_set.hierarchy) else: # OR assign resource by considering only resource_id (no hierarchy) # and only one type of resource (hy_level_nbs, constraints) = hy_res_rqts[0] (h_name, nb_asked_res) = hy_level_nbs[0] itvs_avail = intersec(constraints, res_itvs) ids_avail = itvs2ids(itvs_avail) if len(ids_avail) < nb_asked_res: itvs = [] else: itvs = unordered_ids2itvs(ids_avail[:nb_asked_res]) if (itvs != []): job.moldable_id = mld_id job.res_set = itvs assigned_jobs[job.id] = job res_itvs = sub_intervals(res_itvs, itvs) else: logger.debug("Not enough available resources, it's a FIFO scheduler, we stop here.") break # # Save assignement # logger.info("save assignement") plt.save_assigns(assigned_jobs, resource_set) else: logger.info("no waiting jobs")
def test_subintervals_2(): x = [(1, 10), (20, 30)] y = [(2, 5), (24, 27)] r = [(1, 1), (6, 10), (20, 23), (28, 30)] assert sub_intervals(x, y) == r
def test_subintervals_1(): x = [(1, 10)] y = [(2, 5)] r = [(1, 1), (6, 10)] assert sub_intervals(x, y) == r