def call_external_scheduler(binpath, scheduled_jobs, all_slot_sets,
resource_set, job_security_time, queue,
initial_time_sec, initial_time_sql): # pragma: no cover
cmd_scheduler = binpath + "schedulers/" + queue.scheduler_policy
child_launched = True
# TODO TO CONFIRM
sched_exit_code = 0
sched_signal_num = 0
sched_dumped_core = 0
try:
child = Popen([cmd_scheduler, queue.name, str(
initial_time_sec), initial_time_sql], stdout=subprocess.PIPE)
for line in iter(child.stdout.readline, ''):
logger.debug("Read on the scheduler output:" + str(line.rstrip()))
# TODO SCHEDULER_LAUNCHER_OPTIMIZATION
# if
# ((get_conf_with_default_param('SCHEDULER_LAUNCHER_OPTIMIZATION',
# 'yes') eq 'yes') and
child.wait()
rc = child.returncode
sched_exit_code, sched_signal_num, sched_dumped_core = rc >> 8, rc & 0x7f, bool(
rc & 0x80)
except OSError as e:
child_launched = False
logger.warn(str(e) + " Cannot run: " + cmd_scheduler + " " + queue.name + " " +
str(initial_time_sec) + " " + initial_time_sql)
if (not child_launched) or (sched_signal_num != 0) or (sched_dumped_core != 0):
logger.error("Execution of " + queue.scheduler_policy +
" failed, inactivating queue " + queue.name + " (see `oarnotify')")
# stop queue
db.query(Queue).filter(Queue.name == queue.name).update({'state': 'notActive'})
if sched_exit_code != 0:
logger.error("Scheduler " + queue.scheduler_policy + " returned a bad value: " +
str(sched_exit_code) + ". Inactivating queue " + queue.scheduler_policy +
" (see `oarnotify')")
# stop queue
db.query(Queue).filter(Queue.name == queue.name).update({'state': 'notActive'})
# retrieve jobs and assignement decision from previous scheduling step
scheduled_jobs = get_after_sched_no_AR_jobs(queue.name, resource_set,
job_security_time, initial_time_sec)
if scheduled_jobs != []:
if queue.name == 'besteffort':
filter_besteffort = False
else:
filter_besteffort = True
set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs,
job_security_time, initial_time_sec,
filter_besteffort)
def call_internal_scheduler(plt, scheduled_jobs, all_slot_sets, job_security_time,
queue, now):
# Place running besteffort jobs if their queue is considered
if queue.name == 'besteffort':
set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs,
job_security_time, now,
False, True)
internal_schedule_cycle(plt, now, all_slot_sets, job_security_time,
queue.name)
def test_schedule_placeholder_prev_sched():
res = [(1, 32)]
ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
all_ss = {"default": ss}
hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}
j1 = JobPseudo(
id=1,
types={},
deps=[],
key_cache={},
res_set=[(7, 27)],
start_time=200,
walltime=150,
mld_res_rqts=[(1, 60, [([("node", 2)], res[:])])],
ts=False,
ph=1,
ph_name="yop",
)
j2 = JobPseudo(
id=2, types={}, deps=[], key_cache={}, mld_res_rqts=[(1, 150, [([("node", 4)], res[:])])], ts=False, ph=0
)
j3 = JobPseudo(
id=3,
types={},
deps=[],
key_cache={},
mld_res_rqts=[(1, 500, [([("node", 2)], res[:])])],
ts=False,
ph=2,
ph_name="yop",
)
# pdb.set_trace()
set_slots_with_prev_scheduled_jobs(all_ss, [j1], 20)
all_ss["default"].show_slots()
# pdb.set_trace()
schedule_id_jobs_ct(all_ss, {2: j2, 3: j3}, hy, [2, 3], 20)
print("j1.start_time:", j1.start_time, "j2.start_time:", j2.start_time, " j3.start_time:", j3.start_time)
all_ss["default"].show_slots()
# pdb.set_trace()
# assert j3.start_time == 150
assert j3.res_set == [(1, 16)]
def test_schedule_w_temporally_fragmented_container():
res = [(1, 32)]
ss = SlotSet(Slot(1, 0, 0, res, 0, 5000))
all_ss = {"default": ss}
hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}
j1 = JobPseudo(
id=1,
types={"container": "yop"},
deps=[],
key_cache={},
res_set=[(7, 32)],
start_time=200,
walltime=50,
ts=False,
ph=0,
)
j2 = JobPseudo(
id=2,
types={"container": "yop"},
deps=[],
key_cache={},
res_set=[(15, 25)],
start_time=1000,
walltime=200,
ts=False,
ph=0,
)
j3 = JobPseudo(
id=3,
types={"inner": "yop"},
deps=[],
key_cache={},
mld_res_rqts=[(1, 100, [([("node", 1)], res[:])])],
ts=False,
ph=0,
)
set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 20)
all_ss["yop"].show_slots()
schedule_id_jobs_ct(all_ss, {3: j3}, hy, [3], 20)
all_ss["yop"].show_slots()
assert j3.start_time == 1000
assert j3.res_set == [(17, 24)]
def test_schedule_container_prev_sched():
res = [(1, 32)]
ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
all_ss = {"default": ss}
hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}
j1 = JobPseudo(
id=1,
types={"container": ""},
deps=[],
key_cache={},
res_set=[(7, 27)],
start_time=200,
walltime=150,
mld_res_rqts=[(1, 60, [([("node", 2)], res[:])])],
ts=False,
ph=0,
)
j2 = JobPseudo(
id=2,
types={"inner": "1"},
deps=[],
key_cache={},
res_set=[(9, 16)],
start_time=210,
walltime=70,
mld_res_rqts=[(1, 30, [([("node", 1)], res[:])])],
ts=False,
ph=0,
)
j3 = JobPseudo(
id=3,
types={"inner": "1"},
deps=[],
key_cache={},
mld_res_rqts=[(1, 30, [([("node", 1)], res[:])])],
ts=False,
ph=0,
)
set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 20)
schedule_id_jobs_ct(all_ss, {3: j3}, hy, [3], 20)
assert j3.start_time == 200
assert j3.res_set == [(17, 24)]
def test_set_slots_with_prev_scheduled_jobs_1():
v = [
(1, 4, [(1, 32)]),
(5, 14, [(1, 9), (21, 32)]),
(15, 29, [(1, 32)]),
(30, 49, [(1, 4), (16, 19), (29, 32)]),
(50, 100, [(1, 32)]),
]
j1 = JobPseudo(id=1, start_time=5, walltime=10, res_set=[(10, 20)], types={}, ts=False, ph=0)
j2 = JobPseudo(id=2, start_time=30, walltime=20, res_set=[(5, 15), (20, 28)], types={}, ts=False, ph=0)
ss = SlotSet(Slot(1, 0, 0, [(1, 32)], 1, 100))
all_ss = {"default": ss}
set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 10)
assert compare_slots_val_ref(ss.slots, v) is True
def test_schedule_error_1():
# Be careful you need a deepcopy for resources constraint when declare
res = [(1, 32)]
ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 10000))
all_ss = {"default": ss}
hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}
j2 = JobPseudo(id=2, start_time=0, walltime=30, res_set=[(1, 8)], types={}, ts=False, ph=0)
set_slots_with_prev_scheduled_jobs(all_ss, [j2], 10)
j4 = JobPseudo(
id=4, types={}, deps=[], key_cache={}, mld_res_rqts=[(1, 60, [([("node", 2)], deepcopy(res))])], ts=False, ph=0
)
schedule_id_jobs_ct(all_ss, {4: j4}, hy, [4], 5)
assert j4.res_set == [(9, 24)]
def test_quotas_four_jobs_rule_1():
config['QUOTAS'] = 'yes'
# quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
# global quotas_rules
qts.quotas_rules = {('*', '*', '*', '/'): [16, -1, -1],
('*', 'yop', '*', '*'): [-1, 1, -1]}
res = [(1, 32)]
rs.default_resource_itvs = deepcopy(res)
ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 10000))
all_ss = {"default": ss}
hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}
j1 = JobPseudo(id=1, start_time=0, walltime=20,
queue='default', user='******', project='',
res_set=[(9, 24)], types={}, ts=False, ph=0)
j2 = JobPseudo(id=2, start_time=0, walltime=50,
queue='default', user='******', project='yop',
res_set=[(1, 8)])
j3 = JobPseudo(id=3, queue='default', user='******', project='')
j3.simple_req(('node', 1), 10, res)
j4 = JobPseudo(id=4, queue='default', user='******', project='yop')
j4.simple_req(('node', 1), 60, res)
set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 5)
ss.show_slots()
# pdb.set_trace()
schedule_id_jobs_ct(all_ss, {3: j3, 4: j4}, hy, [3, 4], 5)
print(j3.start_time, j4.start_time)
assert j3.start_time == 20
assert j3.res_set == [(9, 16)]
assert j4.start_time == 50
assert j4.res_set == [(1, 8)]
def gantt_init_with_running_jobs(plt, initial_time_sec, job_security_time):
#
# Determine Global Resource Intervals and Initial Slot
#
resource_set = plt.resource_set()
initial_slot_set = SlotSet((resource_set.roid_itvs, initial_time_sec))
logger.debug("Processing of processing of already handled reservations")
accepted_ar_jids, accepted_ar_jobs = \
get_waiting_reservations_already_scheduled(resource_set, job_security_time)
gantt_flush_tables(accepted_ar_jids)
# TODO Can we remove this step, below ???
# why don't use: assigned_resources and job start_time ??? in get_scheduled_jobs ???
logger.debug("Processing of current jobs")
current_jobs = get_jobs_in_multiple_states(['Running', 'toLaunch', 'Launching',
'Finishing', 'Suspended', 'Resuming'],
resource_set)
plt.save_assigns(current_jobs, resource_set)
#
# Resource availabilty (Available_upto field) is integrated through pseudo job
#
pseudo_jobs = []
for t_avail_upto in sorted(resource_set.available_upto.keys()):
itvs = resource_set.available_upto[t_avail_upto]
j = JobPseudo()
j.start_time = t_avail_upto
j.walltime = MAX_TIME - t_avail_upto
j.res_set = itvs
j.ts = False
j.ph = NO_PLACEHOLDER
pseudo_jobs.append(j)
if pseudo_jobs != []:
initial_slot_set.split_slots_jobs(pseudo_jobs)
#
# Get already scheduled jobs advanced reservations and jobs from more higher priority queues
#
# TODO?: Remove resources of the type specified in
# SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
scheduled_jobs = plt.get_scheduled_jobs(
resource_set, job_security_time, initial_time_sec)
# retrieve ressources used by besteffort jobs
besteffort_rid2job = {}
for job in scheduled_jobs:
# print("job.id:", job.id, job.queue_name, job.types, job.res_set, job.start_time)
if 'besteffort' in job.types:
for r_id in itvs2ids(job.res_set):
besteffort_rid2job[r_id] = job
# Create and fill gantt
all_slot_sets = {'default': initial_slot_set}
if scheduled_jobs != []:
filter_besteffort = True
set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs,
job_security_time, initial_time_sec,
filter_besteffort)
return (all_slot_sets, scheduled_jobs, besteffort_rid2job)
def schedule_cycle(plt, now, queue="default"):
logger.info("Begin scheduling....now: " + str(now) + ", queue: " + queue)
#
# Retrieve waiting jobs
#
waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)
if nb_waiting_jobs > 0:
logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs))
for jid in waiting_jids:
logger.debug("waiting_jid: " + str(jid))
job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])
#
# Determine Global Resource Intervals and Initial Slot
#
resource_set = plt.resource_set()
initial_slot_set = SlotSet((resource_set.roid_itvs, now))
#
# Resource availabilty (Available_upto field) is integrated through pseudo job
#
pseudo_jobs = []
for t_avail_upto in sorted(resource_set.available_upto.keys()):
itvs = resource_set.available_upto[t_avail_upto]
j = JobPseudo()
# print t_avail_upto, max_time - t_avail_upto, itvs
j.start_time = t_avail_upto
j.walltime = MAX_TIME - t_avail_upto
j.res_set = itvs
j.ts = False
j.ph = NO_PLACEHOLDER
pseudo_jobs.append(j)
if pseudo_jobs != []:
initial_slot_set.split_slots_jobs(pseudo_jobs)
#
# Get additional waiting jobs' data
#
plt.get_data_jobs(
waiting_jobs, waiting_jids, resource_set, job_security_time)
# Job sorting (karma and advanced)
waiting_ordered_jids = karma_job_sorting(queue, now, waiting_jids, waiting_jobs, plt)
#
# Get already scheduled jobs advanced reservations and jobs from more higher priority queues
#
scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)
all_slot_sets = {'default': initial_slot_set}
if scheduled_jobs != []:
if queue == 'besteffort':
filter_besteffort = False
else:
filter_besteffort = True
set_slots_with_prev_scheduled_jobs(all_slot_sets,
scheduled_jobs,
job_security_time,
now,
filter_besteffort)
#
# Scheduled
#
schedule_id_jobs_ct(all_slot_sets,
waiting_jobs,
resource_set.hierarchy,
waiting_ordered_jids,
job_security_time)
#
# Save assignement
#
logger.info("save assignement")
plt.save_assigns(waiting_jobs, resource_set)
else:
logger.info("no waiting jobs")
