def test_bug_split_slots(): v = [(20, 69, [(31, 32)]), (70, 79, [(1, 15), (31, 32)]), (80, 2147483599, [(1, 32)]), (2147483600, 2147483647, []) ] # res = [(1, 32)] s1 = Slot(1, 0, 4, [(16, 32)], 20, 69) s2 = Slot(2, 1, 0, [], 2147483600, 2147483647) s4 = Slot(4, 1, 2, [(1, 32)], 70, 2147483599) slts = dict(((s.id, s) for s in [s1, s2, s4])) ss = SlotSet(slts) j2 = JobPseudo(id=2, start_time=20, walltime=60, res_set=[(16, 30)], ts=False, ph=0) ss.split_slots(1, 4, j2) assert compare_slots_val_ref(ss.slots, v)
def test_add_split_slots_jobs_2_jobs_2(): v = [(10, 19, []), (20, 99, [(40, 50)]), (100, 129, [(10, 20), (40, 50)]), (130, 219, [(40, 50)]), (220, MAX_TIME, []), ] ss = SlotSet(([], 10)) j1 = JobPseudo(id=1, start_time=100, walltime=30, res_set=[(10, 20)], ts=False, ph=0) j2 = JobPseudo(id=2, start_time=20, walltime=200, res_set=[(40, 50)], ts=False, ph=0) ss.split_slots_jobs([j2, j1], False) assert compare_slots_val_ref(ss.slots, v)
def test_assign_resources_mld_job_split_slots_2(): v = [(0, 59, [(17, 21)]), (60, 100, [(1, 32)])] res = [(1, 32)] ss = SlotSet(Slot(1, 0, 0, res, 0, 100)) hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]], "switch": [[(1, 16)], [(17, 21)]], "gpu": [[(22, 32)]]} j1 = JobPseudo(id=1, key_cache={}, mld_res_rqts=[(1, 60, [([("node", 2)], res), ([("gpu", 1)], res)])]) assign_resources_mld_job_split_slots(ss, j1, hy, -1) ss.show_slots() assert compare_slots_val_ref(ss.slots, v)
def test_split_slots_bc(): v = [(1, 10, [(1, 9), (21, 32)]), (11, 20, [(1, 32)])] j1 = JobPseudo(id=1, start_time=1, walltime=10, res_set=[(10, 20)], moldable_id=1, ts=False, ph=0) ss = SlotSet(Slot(1, 0, 0, [(1, 32)], 1, 20)) ss.split_slots(1, 1, j1) assert compare_slots_val_ref(ss.slots, v)
def test_add_split_slots_jobs_one_job(): v = [(10, 14, [(10, 50)]), (15, MAX_TIME, [])] ss = SlotSet(([], 10)) j = JobPseudo(id=1, start_time=5, walltime=10, res_set=[(10, 50)], ts=False, ph=0) ss.split_slots_jobs([j], False) assert compare_slots_val_ref(ss.slots, v)
def test_quotas_four_jobs_rule_1(): config['QUOTAS'] = 'yes' # quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]}) # global quotas_rules qts.quotas_rules = {('*', '*', '*', '/'): [16, -1, -1], ('*', 'yop', '*', '*'): [-1, 1, -1]} res = [(1, 32)] rs.default_resource_itvs = deepcopy(res) ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 10000)) all_ss = {"default": ss} hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]} j1 = JobPseudo(id=1, start_time=0, walltime=20, queue='default', user='******', project='', res_set=[(9, 24)], types={}, ts=False, ph=0) j2 = JobPseudo(id=2, start_time=0, walltime=50, queue='default', user='******', project='yop', res_set=[(1, 8)]) j3 = JobPseudo(id=3, queue='default', user='******', project='') j3.simple_req(('node', 1), 10, res) j4 = JobPseudo(id=4, queue='default', user='******', project='yop') j4.simple_req(('node', 1), 60, res) set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 5) ss.show_slots() # pdb.set_trace() schedule_id_jobs_ct(all_ss, {3: j3, 4: j4}, hy, [3, 4], 5) print(j3.start_time, j4.start_time) assert j3.start_time == 20 assert j3.res_set == [(9, 16)] assert j4.start_time == 50 assert j4.res_set == [(1, 8)]
def gantt_init_with_running_jobs(plt, initial_time_sec, job_security_time): # # Determine Global Resource Intervals and Initial Slot # resource_set = plt.resource_set() initial_slot_set = SlotSet((resource_set.roid_itvs, initial_time_sec)) logger.debug("Processing of processing of already handled reservations") accepted_ar_jids, accepted_ar_jobs = \ get_waiting_reservations_already_scheduled(resource_set, job_security_time) gantt_flush_tables(accepted_ar_jids) # TODO Can we remove this step, below ??? # why don't use: assigned_resources and job start_time ??? in get_scheduled_jobs ??? logger.debug("Processing of current jobs") current_jobs = get_jobs_in_multiple_states(['Running', 'toLaunch', 'Launching', 'Finishing', 'Suspended', 'Resuming'], resource_set) plt.save_assigns(current_jobs, resource_set) # # Resource availabilty (Available_upto field) is integrated through pseudo job # pseudo_jobs = [] for t_avail_upto in sorted(resource_set.available_upto.keys()): itvs = resource_set.available_upto[t_avail_upto] j = JobPseudo() j.start_time = t_avail_upto j.walltime = MAX_TIME - t_avail_upto j.res_set = itvs j.ts = False j.ph = NO_PLACEHOLDER pseudo_jobs.append(j) if pseudo_jobs != []: initial_slot_set.split_slots_jobs(pseudo_jobs) # # Get already scheduled jobs advanced reservations and jobs from more higher priority queues # # TODO?: Remove resources of the type specified in # SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE scheduled_jobs = plt.get_scheduled_jobs( resource_set, job_security_time, initial_time_sec) # retrieve ressources used by besteffort jobs besteffort_rid2job = {} for job in scheduled_jobs: # print("job.id:", job.id, job.queue_name, job.types, job.res_set, job.start_time) if 'besteffort' in job.types: for r_id in itvs2ids(job.res_set): besteffort_rid2job[r_id] = job # Create and fill gantt all_slot_sets = {'default': initial_slot_set} if scheduled_jobs != []: filter_besteffort = True set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs, job_security_time, initial_time_sec, filter_besteffort) return (all_slot_sets, scheduled_jobs, besteffort_rid2job)
def schedule_cycle(plt, queue='default'): now = plt.get_time() logger.info('Begin scheduling....', now) # # Retrieve waiting jobs # waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue) logger.info(waiting_jobs, waiting_jids, nb_waiting_jobs) if nb_waiting_jobs > 0: # # Determine Global Resource Intervals and Initial Slot # resource_set = plt.resource_set() initial_slot_set = SlotSet((resource_set.roid_itvs, now)) # # Resource availabilty (Available_upto field) is integrated through pseudo job # pseudo_jobs = [] for t_avail_upto in sorted(resource_set.available_upto.keys()): itvs = resource_set.available_upto[t_avail_upto] j = JobPseudo() # logger.info(t_avail_upto, MAX_TIME - t_avail_upto, itvs) j.start_time = t_avail_upto j.walltime = MAX_TIME - t_avail_upto j.res_set = itvs j.ts = False j.ph = NO_PLACEHOLDER pseudo_jobs.append(j) if pseudo_jobs != []: initial_slot_set.split_slots_jobs(pseudo_jobs) # # Get additional waiting jobs' data # job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"]) plt.get_data_jobs(waiting_jobs, waiting_jids, resource_set, job_security_time) # # Get already scheduled jobs advanced reservations and jobs from more higher priority queues # scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now) if scheduled_jobs != []: initial_slot_set.split_slots_jobs(scheduled_jobs) # initial_slot_set.show_slots() all_slot_sets = {"default": initial_slot_set} # # Scheduled # schedule_id_jobs_ct(all_slot_sets, waiting_jobs, resource_set.hierarchy, waiting_jids, 0) # # Save assignement # plt.save_assigns(waiting_jobs, resource_set) else: logger.info("no waiting jobs")
def schedule_cycle(plt, now, queue="default"): logger.info("Begin scheduling....now: " + str(now) + ", queue: " + queue) # # Retrieve waiting jobs # waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue) if nb_waiting_jobs > 0: logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs)) for jid in waiting_jids: logger.debug("waiting_jid: " + str(jid)) job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"]) # # Determine Global Resource Intervals and Initial Slot # resource_set = plt.resource_set() initial_slot_set = SlotSet((resource_set.roid_itvs, now)) # # Resource availabilty (Available_upto field) is integrated through pseudo job # pseudo_jobs = [] for t_avail_upto in sorted(resource_set.available_upto.keys()): itvs = resource_set.available_upto[t_avail_upto] j = JobPseudo() # print t_avail_upto, max_time - t_avail_upto, itvs j.start_time = t_avail_upto j.walltime = MAX_TIME - t_avail_upto j.res_set = itvs j.ts = False j.ph = NO_PLACEHOLDER pseudo_jobs.append(j) if pseudo_jobs != []: initial_slot_set.split_slots_jobs(pseudo_jobs) # # Get additional waiting jobs' data # plt.get_data_jobs( waiting_jobs, waiting_jids, resource_set, job_security_time) # Job sorting (karma and advanced) waiting_ordered_jids = karma_job_sorting(queue, now, waiting_jids, waiting_jobs, plt) # # Get already scheduled jobs advanced reservations and jobs from more higher priority queues # scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now) all_slot_sets = {'default': initial_slot_set} if scheduled_jobs != []: if queue == 'besteffort': filter_besteffort = False else: filter_besteffort = True set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs, job_security_time, now, filter_besteffort) # # Scheduled # schedule_id_jobs_ct(all_slot_sets, waiting_jobs, resource_set.hierarchy, waiting_ordered_jids, job_security_time) # # Save assignement # logger.info("save assignement") plt.save_assigns(waiting_jobs, resource_set) else: logger.info("no waiting jobs")