def test_add_split_slots_jobs_2_jobs_2():
v = [(10, 19, []),
(20, 99, [(40, 50)]),
(100, 129, [(10, 20), (40, 50)]),
(130, 219, [(40, 50)]),
(220, MAX_TIME, []),
]
ss = SlotSet(([], 10))
j1 = JobPseudo(id=1,
start_time=100,
walltime=30,
res_set=[(10, 20)],
ts=False,
ph=0)
j2 = JobPseudo(id=2,
start_time=20,
walltime=200,
res_set=[(40, 50)],
ts=False,
ph=0)
ss.split_slots_jobs([j2, j1], False)
assert compare_slots_val_ref(ss.slots, v)
def test_add_split_slots_jobs_one_job():
v = [(10, 14, [(10, 50)]), (15, MAX_TIME, [])]
ss = SlotSet(([], 10))
j = JobPseudo(id=1,
start_time=5,
walltime=10,
res_set=[(10, 50)],
ts=False,
ph=0)
ss.split_slots_jobs([j], False)
assert compare_slots_val_ref(ss.slots, v)
def gantt_init_with_running_jobs(plt, initial_time_sec, job_security_time):
#
# Determine Global Resource Intervals and Initial Slot
#
resource_set = plt.resource_set()
initial_slot_set = SlotSet((resource_set.roid_itvs, initial_time_sec))
logger.debug("Processing of processing of already handled reservations")
accepted_ar_jids, accepted_ar_jobs = \
get_waiting_reservations_already_scheduled(resource_set, job_security_time)
gantt_flush_tables(accepted_ar_jids)
# TODO Can we remove this step, below ???
# why don't use: assigned_resources and job start_time ??? in get_scheduled_jobs ???
logger.debug("Processing of current jobs")
current_jobs = get_jobs_in_multiple_states(['Running', 'toLaunch', 'Launching',
'Finishing', 'Suspended', 'Resuming'],
resource_set)
plt.save_assigns(current_jobs, resource_set)
#
# Resource availabilty (Available_upto field) is integrated through pseudo job
#
pseudo_jobs = []
for t_avail_upto in sorted(resource_set.available_upto.keys()):
itvs = resource_set.available_upto[t_avail_upto]
j = JobPseudo()
j.start_time = t_avail_upto
j.walltime = MAX_TIME - t_avail_upto
j.res_set = itvs
j.ts = False
j.ph = NO_PLACEHOLDER
pseudo_jobs.append(j)
if pseudo_jobs != []:
initial_slot_set.split_slots_jobs(pseudo_jobs)
#
# Get already scheduled jobs advanced reservations and jobs from more higher priority queues
#
# TODO?: Remove resources of the type specified in
# SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
scheduled_jobs = plt.get_scheduled_jobs(
resource_set, job_security_time, initial_time_sec)
# retrieve ressources used by besteffort jobs
besteffort_rid2job = {}
for job in scheduled_jobs:
# print("job.id:", job.id, job.queue_name, job.types, job.res_set, job.start_time)
if 'besteffort' in job.types:
for r_id in itvs2ids(job.res_set):
besteffort_rid2job[r_id] = job
# Create and fill gantt
all_slot_sets = {'default': initial_slot_set}
if scheduled_jobs != []:
filter_besteffort = True
set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs,
job_security_time, initial_time_sec,
filter_besteffort)
return (all_slot_sets, scheduled_jobs, besteffort_rid2job)
def schedule_cycle(plt, queue='default'):
now = plt.get_time()
logger.info('Begin scheduling....', now)
#
# Retrieve waiting jobs
#
waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)
logger.info(waiting_jobs, waiting_jids, nb_waiting_jobs)
if nb_waiting_jobs > 0:
#
# Determine Global Resource Intervals and Initial Slot
#
resource_set = plt.resource_set()
initial_slot_set = SlotSet((resource_set.roid_itvs, now))
#
# Resource availabilty (Available_upto field) is integrated through pseudo job
#
pseudo_jobs = []
for t_avail_upto in sorted(resource_set.available_upto.keys()):
itvs = resource_set.available_upto[t_avail_upto]
j = JobPseudo()
# logger.info(t_avail_upto, MAX_TIME - t_avail_upto, itvs)
j.start_time = t_avail_upto
j.walltime = MAX_TIME - t_avail_upto
j.res_set = itvs
j.ts = False
j.ph = NO_PLACEHOLDER
pseudo_jobs.append(j)
if pseudo_jobs != []:
initial_slot_set.split_slots_jobs(pseudo_jobs)
#
# Get additional waiting jobs' data
#
job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])
plt.get_data_jobs(waiting_jobs, waiting_jids, resource_set, job_security_time)
#
# Get already scheduled jobs advanced reservations and jobs from more higher priority queues
#
scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)
if scheduled_jobs != []:
initial_slot_set.split_slots_jobs(scheduled_jobs)
# initial_slot_set.show_slots()
all_slot_sets = {"default": initial_slot_set}
#
# Scheduled
#
schedule_id_jobs_ct(all_slot_sets,
waiting_jobs,
resource_set.hierarchy,
waiting_jids,
0)
#
# Save assignement
#
plt.save_assigns(waiting_jobs, resource_set)
else:
logger.info("no waiting jobs")
def schedule_cycle(plt, now, queue="default"):
logger.info("Begin scheduling....now: " + str(now) + ", queue: " + queue)
#
# Retrieve waiting jobs
#
waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)
if nb_waiting_jobs > 0:
logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs))
for jid in waiting_jids:
logger.debug("waiting_jid: " + str(jid))
job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])
#
# Determine Global Resource Intervals and Initial Slot
#
resource_set = plt.resource_set()
initial_slot_set = SlotSet((resource_set.roid_itvs, now))
#
# Resource availabilty (Available_upto field) is integrated through pseudo job
#
pseudo_jobs = []
for t_avail_upto in sorted(resource_set.available_upto.keys()):
itvs = resource_set.available_upto[t_avail_upto]
j = JobPseudo()
# print t_avail_upto, max_time - t_avail_upto, itvs
j.start_time = t_avail_upto
j.walltime = MAX_TIME - t_avail_upto
j.res_set = itvs
j.ts = False
j.ph = NO_PLACEHOLDER
pseudo_jobs.append(j)
if pseudo_jobs != []:
initial_slot_set.split_slots_jobs(pseudo_jobs)
#
# Get additional waiting jobs' data
#
plt.get_data_jobs(
waiting_jobs, waiting_jids, resource_set, job_security_time)
# Job sorting (karma and advanced)
waiting_ordered_jids = karma_job_sorting(queue, now, waiting_jids, waiting_jobs, plt)
#
# Get already scheduled jobs advanced reservations and jobs from more higher priority queues
#
scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)
all_slot_sets = {'default': initial_slot_set}
if scheduled_jobs != []:
if queue == 'besteffort':
filter_besteffort = False
else:
filter_besteffort = True
set_slots_with_prev_scheduled_jobs(all_slot_sets,
scheduled_jobs,
job_security_time,
now,
filter_besteffort)
#
# Scheduled
#
schedule_id_jobs_ct(all_slot_sets,
waiting_jobs,
resource_set.hierarchy,
waiting_ordered_jids,
job_security_time)
#
# Save assignement
#
logger.info("save assignement")
plt.save_assigns(waiting_jobs, resource_set)
else:
logger.info("no waiting jobs")
