def test_quotas_one_job_rule_nb_res_1():
config['QUOTAS'] = 'yes'
# quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
# global quotas_rules
qts.quotas_rules = {('*', '*', '*', '/'): [1, -1, -1]}
res = [(1, 32)]
rs.default_resource_itvs = deepcopy(res)
ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 100))
all_ss = {"default": ss}
hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}
j1 = JobPseudo(id=1, queue='default', user='******', project='')
j1.simple_req(('node', 2), 60, res)
schedule_id_jobs_ct(all_ss, {1: j1}, hy, [1], 20)
print(j1.start_time)
assert j1.res_set == []
def test_quotas_two_jobs_job_type_proc():
config['QUOTAS'] = 'yes'
_, quotas_file_name = mkstemp()
config['QUOTAS_FILE'] = quotas_file_name
# quotas_file = open(quotas_file_name, 'w')
with open(config['QUOTAS_FILE'], 'w', encoding="utf-8") as quotas_fd:
quotas_fd.write('{"quotas": {"*,*,yop,*": [-1,1,-1]}, "quotas_job_types": ["yop"]}')
qts.load_quotas_rules()
print(qts.quotas_rules, qts.quotas_job_types)
res = [(1, 32)]
rs.default_resource_itvs = res
ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
all_ss = {"default": ss}
hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}
j1 = JobPseudo(id=1, queue='default', user='******', project='', types={'yop'})
j1.simple_req(('node', 1), 50, res)
j2 = JobPseudo(id=2, queue='default', user='******', project='', types={'yop'})
j2.simple_req(('node', 1), 50, res)
schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 20)
print(j1.start_time, j2.start_time)
assert j1.start_time == 0
assert j2.start_time == 50
def test_quotas_four_jobs_rule_1():
config['QUOTAS'] = 'yes'
# quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
# global quotas_rules
qts.quotas_rules = {('*', '*', '*', '/'): [16, -1, -1],
('*', 'yop', '*', '*'): [-1, 1, -1]}
res = [(1, 32)]
rs.default_resource_itvs = deepcopy(res)
ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 10000))
all_ss = {"default": ss}
hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}
j1 = JobPseudo(id=1, start_time=0, walltime=20,
queue='default', user='******', project='',
res_set=[(9, 24)], types={}, ts=False, ph=0)
j2 = JobPseudo(id=2, start_time=0, walltime=50,
queue='default', user='******', project='yop',
res_set=[(1, 8)])
j3 = JobPseudo(id=3, queue='default', user='******', project='')
j3.simple_req(('node', 1), 10, res)
j4 = JobPseudo(id=4, queue='default', user='******', project='yop')
j4.simple_req(('node', 1), 60, res)
set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 5)
ss.show_slots()
# pdb.set_trace()
schedule_id_jobs_ct(all_ss, {3: j3, 4: j4}, hy, [3, 4], 5)
print(j3.start_time, j4.start_time)
assert j3.start_time == 20
assert j3.res_set == [(9, 16)]
assert j4.start_time == 50
assert j4.res_set == [(1, 8)]
def gantt_init_with_running_jobs(plt, initial_time_sec, job_security_time):
#
# Determine Global Resource Intervals and Initial Slot
#
resource_set = plt.resource_set()
initial_slot_set = SlotSet((resource_set.roid_itvs, initial_time_sec))
logger.debug("Processing of processing of already handled reservations")
accepted_ar_jids, accepted_ar_jobs = \
get_waiting_reservations_already_scheduled(resource_set, job_security_time)
gantt_flush_tables(accepted_ar_jids)
# TODO Can we remove this step, below ???
# why don't use: assigned_resources and job start_time ??? in get_scheduled_jobs ???
logger.debug("Processing of current jobs")
current_jobs = get_jobs_in_multiple_states(['Running', 'toLaunch', 'Launching',
'Finishing', 'Suspended', 'Resuming'],
resource_set)
plt.save_assigns(current_jobs, resource_set)
#
# Resource availabilty (Available_upto field) is integrated through pseudo job
#
pseudo_jobs = []
for t_avail_upto in sorted(resource_set.available_upto.keys()):
itvs = resource_set.available_upto[t_avail_upto]
j = JobPseudo()
j.start_time = t_avail_upto
j.walltime = MAX_TIME - t_avail_upto
j.res_set = itvs
j.ts = False
j.ph = NO_PLACEHOLDER
pseudo_jobs.append(j)
if pseudo_jobs != []:
initial_slot_set.split_slots_jobs(pseudo_jobs)
#
# Get already scheduled jobs advanced reservations and jobs from more higher priority queues
#
# TODO?: Remove resources of the type specified in
# SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
scheduled_jobs = plt.get_scheduled_jobs(
resource_set, job_security_time, initial_time_sec)
# retrieve ressources used by besteffort jobs
besteffort_rid2job = {}
for job in scheduled_jobs:
# print("job.id:", job.id, job.queue_name, job.types, job.res_set, job.start_time)
if 'besteffort' in job.types:
for r_id in itvs2ids(job.res_set):
besteffort_rid2job[r_id] = job
# Create and fill gantt
all_slot_sets = {'default': initial_slot_set}
if scheduled_jobs != []:
filter_besteffort = True
set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs,
job_security_time, initial_time_sec,
filter_besteffort)
return (all_slot_sets, scheduled_jobs, besteffort_rid2job)
def schedule_cycle(plt, queue='default'):
now = plt.get_time()
logger.info('Begin scheduling....', now)
#
# Retrieve waiting jobs
#
waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)
logger.info(waiting_jobs, waiting_jids, nb_waiting_jobs)
if nb_waiting_jobs > 0:
#
# Determine Global Resource Intervals and Initial Slot
#
resource_set = plt.resource_set()
initial_slot_set = SlotSet((resource_set.roid_itvs, now))
#
# Resource availabilty (Available_upto field) is integrated through pseudo job
#
pseudo_jobs = []
for t_avail_upto in sorted(resource_set.available_upto.keys()):
itvs = resource_set.available_upto[t_avail_upto]
j = JobPseudo()
# logger.info(t_avail_upto, MAX_TIME - t_avail_upto, itvs)
j.start_time = t_avail_upto
j.walltime = MAX_TIME - t_avail_upto
j.res_set = itvs
j.ts = False
j.ph = NO_PLACEHOLDER
pseudo_jobs.append(j)
if pseudo_jobs != []:
initial_slot_set.split_slots_jobs(pseudo_jobs)
#
# Get additional waiting jobs' data
#
job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])
plt.get_data_jobs(waiting_jobs, waiting_jids, resource_set, job_security_time)
#
# Get already scheduled jobs advanced reservations and jobs from more higher priority queues
#
scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)
if scheduled_jobs != []:
initial_slot_set.split_slots_jobs(scheduled_jobs)
# initial_slot_set.show_slots()
all_slot_sets = {"default": initial_slot_set}
#
# Scheduled
#
schedule_id_jobs_ct(all_slot_sets,
waiting_jobs,
resource_set.hierarchy,
waiting_jids,
0)
#
# Save assignement
#
plt.save_assigns(waiting_jobs, resource_set)
else:
logger.info("no waiting jobs")
def schedule_cycle(plt, now, queue="default"):
logger.info("Begin scheduling....now: " + str(now) + ", queue: " + queue)
#
# Retrieve waiting jobs
#
waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)
if nb_waiting_jobs > 0:
logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs))
for jid in waiting_jids:
logger.debug("waiting_jid: " + str(jid))
job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])
#
# Determine Global Resource Intervals and Initial Slot
#
resource_set = plt.resource_set()
initial_slot_set = SlotSet((resource_set.roid_itvs, now))
#
# Resource availabilty (Available_upto field) is integrated through pseudo job
#
pseudo_jobs = []
for t_avail_upto in sorted(resource_set.available_upto.keys()):
itvs = resource_set.available_upto[t_avail_upto]
j = JobPseudo()
# print t_avail_upto, max_time - t_avail_upto, itvs
j.start_time = t_avail_upto
j.walltime = MAX_TIME - t_avail_upto
j.res_set = itvs
j.ts = False
j.ph = NO_PLACEHOLDER
pseudo_jobs.append(j)
if pseudo_jobs != []:
initial_slot_set.split_slots_jobs(pseudo_jobs)
#
# Get additional waiting jobs' data
#
plt.get_data_jobs(
waiting_jobs, waiting_jids, resource_set, job_security_time)
# Job sorting (karma and advanced)
waiting_ordered_jids = karma_job_sorting(queue, now, waiting_jids, waiting_jobs, plt)
#
# Get already scheduled jobs advanced reservations and jobs from more higher priority queues
#
scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)
all_slot_sets = {'default': initial_slot_set}
if scheduled_jobs != []:
if queue == 'besteffort':
filter_besteffort = False
else:
filter_besteffort = True
set_slots_with_prev_scheduled_jobs(all_slot_sets,
scheduled_jobs,
job_security_time,
now,
filter_besteffort)
#
# Scheduled
#
schedule_id_jobs_ct(all_slot_sets,
waiting_jobs,
resource_set.hierarchy,
waiting_ordered_jids,
job_security_time)
#
# Save assignement
#
logger.info("save assignement")
plt.save_assigns(waiting_jobs, resource_set)
else:
logger.info("no waiting jobs")