def scheduleJobs(self):
print("Sheduling Round")
real_time = time.time()
if self.platform_model == "simu":
schedule_cycle(self.platform, self.env.now, "default")
# retrieve jobs to launch
jids_to_launch = []
for jid, job in iteritems(self.platform.assigned_jobs):
print("job.start_time %s" % job.start_time)
if (job.start_time == self.env.now) and (job.state == "Waiting"):
self.waiting_jids.remove(jid)
jids_to_launch.append(jid)
job.state = "Running"
print("tolaunch: %s" % jid)
self.platform.running_jids.append(jid)
else:
print("call meta_schedule('internal')")
meta_schedule("internal", plt)
result = db.query(Job).filter(Job.state == "toLaunch").order_by(Job.id).all()
for job_db in result:
set_job_state(job_db.id, "Running")
jid = self.db_jid2s_jid[job_db.id]
self.waiting_jids.remove(jid)
jids_to_launch.append(jid)
self.jobs[jid].state = "Running"
print("_tolaunch: %s" % jid)
self.platform.running_jids.append(jid)
print("Ids of jobs to launch: ", *jids_to_launch)
print("Time befort scheduling round: ", self.bs._current_time, self.sched_delay)
# update time
real_sched_time = time.time() - real_time
if self.sched_delay == -1:
self.bs.consume_time(real_sched_time) # TODO
else:
self.bs.consume_time(self.sched_delay)
self.env.now = self.bs._current_time
print("Time after scheduling round: ", self.bs._current_time)
# send to uds
if len(jids_to_launch) > 0:
scheduled_jobs = []
jobs_res = {}
for jid in jids_to_launch:
ds_job = self.jobs[jid].ds_job
res = itvs2batsim_str0(self.jobs[jid].res_set)
scheduled_jobs.append(ds_job)
jobs_res[ds_job.id] = res
self.bs.start_jobs(scheduled_jobs, jobs_res)
def test_suspend_resume_1(monkeypatch):
# now = get_date()
insert_job(res=[(60, [('resource_id=4', "")])], properties="")
meta_schedule('internal')
job = db['Job'].query.one()
print(job.state)
set_job_state(job.id, 'Resuming')
job = db['Job'].query.one()
print(job.state)
meta_schedule('internal')
assert(job.state == 'Resuming')
def test_suspend_resume_2(monkeypatch):
config['JUST_BEFORE_RESUME_EXEC_FILE'] = 'sleep 2'
# now = get_date()
insert_job(res=[(60, [('resource_id=4', "")])], properties="")
meta_schedule('internal')
job = db['Job'].query.one()
print(job.state)
set_job_state(job.id, 'Resuming')
job = db['Job'].query.one()
print(job.state)
meta_schedule('internal')
assert(job.state == 'Resuming')
def onJobCompletion(self, data_storage_job):
data_storage_jobs = [data_storage_job] # TODO vectorize in batsim.py !!!
for job in data_storage_jobs:
jid = self.ujid_l.index(job.id)
self.jobs_completed.append(jid)
if jid in self.platform.running_jids:
self.platform.running_jids.remove(jid)
if self.platform_model == "batsim-db":
set_job_state(self.jobs[jid].db_jid, "Terminated")
self.scheduleJobs()
def test_db_all_in_one_BE_2(monkeypatch):
# TODO TOFINISH
db['Queue'].create(name='besteffort', priority=3, scheduler_policy='kamelot', state='Active')
insert_job(res=[(100, [('resource_id=1', "")])], queue_name='besteffort',
types=['besteffort', "timesharing=*,*"])
meta_schedule('internal')
job = db['Job'].query.one()
set_job_state(job.id, 'Running')
insert_job(res=[(50, [('resource_id=1', "")])], types=["timesharing=*,*"])
meta_schedule('internal')
jobs = db['Job'].query.all()
print(jobs[1].id, jobs[1].state)
# assert (jobs[1].state == 'toLaunch')
assert (jobs[1].state == 'Waiting')
def prepare_job_to_be_launched(job, current_time_sec):
# TODO ???
# my $running_date = $current_time_sec;
# if ($running_date < $job_submission_time){
# $running_date = $job_submission_time;
# }
# OAR::IO::set_running_date_arbitrary($base, $job_id, $running_date);
# OAR::IO::set_assigned_moldable_job($base, $job_id, $moldable_job_id);
# set start_time an for jobs to launch
set_job_start_time_assigned_moldable_id(job.id,
current_time_sec,
job.moldable_id)
# fix resource assignement
add_resource_job_pairs(job.moldable_id)
set_job_state(job.id, 'toLaunch')
notify_to_run_job(job.id)
def sched_loop(self):
nb_completed_jobs = 0
while nb_completed_jobs < self.nb_jobs:
now_float, jobs_submitted, new_jobs_completed = read_bat_msg(self.sock)
# now_str = "10"
# jobs_submitted = [1]
# new_jobs_completed = []
if jobs_submitted:
for jid in jobs_submitted:
self.waiting_jids.add(jid)
if self.platform_model == "batsim-db":
print('set_job_state("Waiting"):', self.jobs[jid].db_jid)
set_job_state(self.jobs[jid].db_jid, "Waiting")
nb_completed_jobs += len(new_jobs_completed)
print("new job completed: %s" % new_jobs_completed)
for jid in new_jobs_completed:
jobs_completed.append(jid)
if jid in self.platform.running_jids:
self.platform.running_jids.remove(jid)
if self.platform_model == "batsim-db":
set_job_state(self.jobs[jid].db_jid, "Terminated")
now = int(now_float)
self.env.now = now # TODO can be remove ???
real_time = time.time()
print("jobs running: %s" % self.platform.running_jids)
print("jobs waiting: %s" % self.waiting_jids)
print("jobs completed: %s" % jobs_completed)
jids_to_launch = []
if self.platform_model == "simu":
print("call schedule_cycle.... %s" % now)
schedule_cycle(self.platform, now, "default")
# retrieve jobs to launch
jids_to_launch = []
for jid, job in iteritems(self.platform.assigned_jobs):
print("job.start_time %s" % job.start_time)
if (job.start_time == now) and (job.state == "Waiting"):
self.waiting_jids.remove(jid)
jids_to_launch.append(jid)
job.state = "Running"
print("tolaunch: %s" % jid)
self.platform.running_jids.append(jid)
else:
print("call meta_schedule('internal')")
meta_schedule("internal", plt)
# Launching phase
# Retrieve job to Launch
result = db.query(Job).filter(Job.state == "toLaunch").order_by(Job.id).all()
for job_db in result:
set_job_state(job_db.id, "Running")
jid = self.db_jid2s_jid[job_db.id]
self.waiting_jids.remove(jid)
jids_to_launch.append(jid)
self.jobs[jid].state = "Running"
print("_tolaunch: %s" % jid)
self.platform.running_jids.append(jid)
real_sched_time = time.time() - real_time
if self.sched_delay == -1:
now_float += real_sched_time
else:
now_float += self.sched_delay
send_bat_msg(self.sock, now_float, jids_to_launch, self.jobs)
def meta_schedule(mode='internal', plt=Platform()):
exit_code = 0
job_security_time = int(config['SCHEDULER_JOB_SECURITY_TIME'])
if ('QUOTAS' in config) and (config['QUOTAS'] == 'yes'):
if 'QUOTAS_FILE' not in config:
config['QUOTAS_FILE'] = './quotas_conf.json'
load_quotas_rules()
tools.init_judas_notify_user()
tools.create_almighty_socket()
logger.debug(
"Retrieve information for already scheduled reservations from \
database before flush (keep assign resources)")
# reservation ??.
initial_time_sec = tools.get_date() # time.time()
initial_time_sql = local_to_sql(initial_time_sec)
current_time_sec = initial_time_sec
current_time_sql = initial_time_sql
gantt_init_results = gantt_init_with_running_jobs(plt, initial_time_sec,
job_security_time)
all_slot_sets, scheduled_jobs, besteffort_rid2jid = gantt_init_results
resource_set = plt.resource_set()
# Path for user of external schedulers
if 'OARDIR' in os.environ:
binpath = os.environ['OARDIR'] + '/'
else:
binpath = '/usr/local/lib/oar/'
logger.warning(
"OARDIR env variable must be defined, " + binpath + " is used by default")
for queue in db.query(Queue).order_by(text('priority DESC')).all():
if queue.state == 'Active':
logger.debug("Queue " + queue.name + ": Launching scheduler " +
queue.scheduler_policy + " at time " + initial_time_sql)
if mode == 'external': # pragma: no cover
call_external_scheduler(binpath, scheduled_jobs, all_slot_sets,
resource_set, job_security_time, queue,
initial_time_sec, initial_time_sql)
else:
call_internal_scheduler(plt, scheduled_jobs, all_slot_sets,
job_security_time, queue, initial_time_sec)
handle_waiting_reservation_jobs(queue.name, resource_set,
job_security_time, current_time_sec)
# handle_new_AR_jobs
check_reservation_jobs(
plt, resource_set, queue.name, all_slot_sets, current_time_sec)
jobs_to_launch, jobs_to_launch_lst, rid2jid_to_launch = get_gantt_jobs_to_launch(resource_set,
job_security_time,
current_time_sec)
if check_besteffort_jobs_to_kill(jobs_to_launch, rid2jid_to_launch,
current_time_sec, besteffort_rid2jid,
resource_set) == 1:
# We must kill some besteffort jobs
tools.notify_almighty('ChState')
exit_code = 2
elif handle_jobs_to_launch(jobs_to_launch_lst, current_time_sec, current_time_sql) == 1:
exit_code = 0
# Update visu gantt tables
update_gantt_visualization()
# Manage dynamic node feature
flag_hulot = False
timeout_cmd = int(config['SCHEDULER_TIMEOUT'])
if ((('SCHEDULER_NODE_MANAGER_SLEEP_CMD' in config) or
((config['ENERGY_SAVING_INTERNAL'] == 'yes') and
('ENERGY_SAVING_NODE_MANAGER_SLEEP_CMD' in config))) and
(('SCHEDULER_NODE_MANAGER_SLEEP_TIME' in config)
and ('SCHEDULER_NODE_MANAGER_IDLE_TIME' in config))):
# Look at nodes that are unused for a duration
idle_duration = int(config['SCHEDULER_NODE_MANAGER_IDLE_TIME'])
sleep_duration = int(config['SCHEDULER_NODE_MANAGER_SLEEP_TIME'])
idle_nodes = search_idle_nodes(current_time_sec)
tmp_time = current_time_sec - idle_duration
node_halt = []
for node, idle_duration in iteritems(idle_nodes):
if idle_duration < tmp_time:
# Search if the node has enough time to sleep
tmp = get_next_job_date_on_node(node)
if (tmp is None) or (tmp - sleep_duration > current_time_sec):
# Search if node has not been woken up recently
wakeup_date = get_last_wake_up_date_of_node(node)
if (wakeup_date is None) or (wakeup_date < tmp_time):
node_halt.append(node)
if node_halt != []:
logger.debug("Powering off some nodes (energy saving): " + str(node_halt))
# Using the built-in energy saving module to shut down nodes
if config['ENERGY_SAVING_INTERNAL'] == 'yes':
if kao_tools.send_to_hulot('HALT', ' '.join(node_halt)):
logger.error("Communication problem with the energy saving module (Hulot)\n")
flag_hulot = 1
else:
# Not using the built-in energy saving module to shut down nodes
cmd = config['SCHEDULER_NODE_MANAGER_SLEEP_CMD']
if kao_tools.fork_and_feed_stdin(cmd, timeout_cmd, node_halt):
logger.error("Command " + cmd + "timeouted (" + str(timeout_cmd)
+ "s) while trying to poweroff some nodes")
if (('SCHEDULER_NODE_MANAGER_SLEEP_CMD' in config) or
((config['ENERGY_SAVING_INTERNAL'] == 'yes') and
('ENERGY_SAVING_NODE_MANAGER_SLEEP_CMD' in config))):
# Get nodes which the scheduler wants to schedule jobs to,
# but which are in the Absent state, to wake them up
wakeup_time = int(config['SCHEDULER_NODE_MANAGER_WAKEUP_TIME'])
nodes = get_gantt_hostname_to_wake_up(current_time_sec, wakeup_time)
if nodes != []:
logger.debug("Awaking some nodes: " + str(nodes))
# Using the built-in energy saving module to wake up nodes
if config['ENERGY_SAVING_INTERNAL'] == 'yes':
if kao_tools.send_to_hulot('WAKEUP', ' '.join(nodes)):
logger.error("Communication problem with the energy saving module (Hulot)")
flag_hulot = 1
else:
# Not using the built-in energy saving module to wake up nodes
cmd = config['SCHEDULER_NODE_MANAGER_WAKE_UP_CMD']
if kao_tools.fork_and_feed_stdin(cmd, timeout_cmd, nodes):
logger.error("Command " + cmd + "timeouted (" + str(timeout_cmd)
+ "s) while trying to wake-up some nodes ")
# Send CHECK signal to Hulot if needed
if not flag_hulot and (config['ENERGY_SAVING_INTERNAL'] == 'yes'):
if kao_tools.send_to_hulot('CHECK', []):
logger.error("Communication problem with the energy saving module (Hulot)")
# Retrieve jobs according to their state and excluding job in 'Waiting' state.
jobs_by_state = get_current_not_waiting_jobs()
#
# Search jobs to resume
#
#
# TODO: TOFINISH
#
if 'Resuming' in jobs_by_state:
logger.warn("Resuming job is NOT ENTIRELY IMPLEMENTED")
for job in jobs_by_state['Resuming']:
other_jobs = get_jobs_on_resuming_job_resources(job.id)
# TODO : look for timesharing other jobs. What do we do?????
if other_jobs == []:
# We can resume the job
logger.debug("[" + str(job.id) + "] Resuming job")
if 'noop' in job.types:
resume_job_action(job.id)
logger.debug("[" + str(job.id) + "] Resume NOOP job OK")
else:
script = config['JUST_BEFORE_RESUME_EXEC_FILE']
timeout = int(config['SUSPEND_RESUME_SCRIPT_TIMEOUT'])
if timeout is None:
timeout = kao_tools.get_default_suspend_resume_script_timeout()
skip = 0
logger.debug("[" + str(job.id) + "] Running post suspend script: `" +
script + " " + str(job.id) + "'")
cmd_str = script + str(job.id)
return_code = -1
try:
return_code = call(cmd_str, shell=True, timeout=timeout)
except TimeoutExpired as e:
logger.error(str(e) + "[" + str(job.id) + "] Suspend script timeouted")
add_new_event('RESUME_SCRIPT_ERROR', job.id, "Suspend script timeouted")
if return_code != 0:
str_error = "[" + str(job.id) + "] Suspend script error, return code = "\
+ str(return_code)
logger.error(str_error)
add_new_event('RESUME_SCRIPT_ERROR', job.id, str_error)
frag_job(job.id)
tools.notify_almighty('Qdel')
skip = 1
cpuset_nodes = None
if 'JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD' in config:
cpuset_field = config['JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD']
else:
cpuset_field = ""
if cpuset_field and (skip == 0):
# TODO
cpuset_name = job.user + "_" + str(job.id)
cpuset_nodes = get_cpuset_values(cpuset_field,
job.assigned_moldable_id)
# TODO
suspend_data_hash = {'name': cpuset_name,
'job_id': job.id,
'oarexec_pid_file':
kao_tools.get_oar_pid_file_name(job.id)}
if cpuset_nodes:
# TODO
taktuk_cmd = config['TAKTUK_CMD']
if 'SUSPEND_RESUME_FILE' in config:
suspend_file = config['SUSPEND_RESUME_FILE']
else:
# TODO
suspend_file = kao_tools.get_default_suspend_resume_file()
#
# TODO: TOFINISH
#
# Notify oarsub -I when they will be launched
for j_info in get_gantt_waiting_interactive_prediction_date():
job_id, job_info_type, job_start_time, job_message = j_info
addr, port = job_info_type.split(':')
new_start_prediction = local_to_sql(job_start_time)
logger.debug("[" + str(job_id) + "] Notifying user of the start prediction: " +
new_start_prediction + "(" + job_message + ")")
tools.notify_tcp_socket(addr, port, "[" + initial_time_sql + "] Start prediction: " +
new_start_prediction + " (" + job_message + ")")
# Run the decisions
# Process "toError" jobs
if 'toError' in jobs_by_state:
for job in jobs_by_state['toError']:
addr, port = job.info_type.split(':')
if job.type == 'INTERACTIVE' or\
(job.type == 'PASSIVE' and job.reservation == 'Scheduled'):
logger.debug("Notify oarsub job (num:" + str(job.id) + ") in error; jobInfo=" +
job.info_type)
nb_sent1 = tools.notify_tcp_socket(addr, port, job.message + '\n')
nb_sent2 = tools.notify_tcp_socket(addr, port, 'BAD JOB' + '\n')
if (nb_sent1 == 0) or (nb_sent2 == 0):
logger.warn(
"Cannot open connection to oarsub client for" + str(job.id))
logger.debug("Set job " + str(job.id) + " to state Error")
set_job_state(job.id, 'Error')
# Process toAckReservation jobs
if 'toAckReservation' in jobs_by_state:
for job in jobs_by_state['toAckReservation']:
addr, port = job.info_type.split(':')
logger.debug(
"Treate job" + str(job.id) + " in toAckReservation state")
nb_sent = tools.notify_tcp_socket(addr, port, 'GOOD RESERVATION' + '\n')
if nb_sent == 0:
logger.warn(
"Frag job " + str(job.id) + ", I cannot notify oarsub for the reservation")
add_new_event('CANNOT_NOTIFY_OARSUB', str(
job.id), "Can not notify oarsub for the job " + str(job.id))
# TODO ???
# OAR::IO::lock_table / OAR::IO::unlock_table($base)
frag_job(job.id)
exit_code = 2
else:
logger.debug("Notify oarsub for a RESERVATION (idJob=" +
str(job.id) + ") --> OK; jobInfo=" + job.info_type)
set_job_state(job.id, 'Waiting')
if ((job.start_time - 1) <= current_time_sec) and (exit_code == 0):
exit_code = 1
# Process toLaunch jobs
if 'toLaunch' in jobs_by_state:
for job in jobs_by_state['toLaunch']:
notify_to_run_job(job.id)
logger.debug("End of Meta Scheduler")
return exit_code
def check_reservation_jobs(plt, resource_set, queue_name, all_slot_sets, current_time_sec):
"""Processing of new Advance Reservations"""
logger.debug("Queue " + queue_name + ": begin processing of new reservations")
ar_jobs_scheduled = {}
ar_jobs, ar_jids, nb_ar_jobs = plt.get_waiting_jobs(
queue_name, 'toSchedule')
logger.debug("nb_ar_jobs:" + str(nb_ar_jobs))
if nb_ar_jobs > 0:
job_security_time = int(config['SCHEDULER_JOB_SECURITY_TIME'])
plt.get_data_jobs(ar_jobs, ar_jids, resource_set, job_security_time)
logger.debug("Try and schedule new reservations")
for jid in ar_jids:
job = ar_jobs[jid]
logger.debug(
"Find resource for Advance Reservation job:" + str(job.id))
# It is a reservation, we take care only of the first moldable job
moldable_id, walltime, hy_res_rqts = job.mld_res_rqts[0]
# test if reservation is too old
if current_time_sec >= (job.start_time + walltime):
logger.warn(
"[" + str(job.id) + "] Canceling job: reservation is too old")
set_job_message(job.id, "Reservation too old")
set_job_state(job.id, 'toError')
continue
else:
if job.start_time < current_time_sec:
# TODO update to DB ????
job.start_time = current_time_sec
ss_name = 'default'
# TODO container
# if 'inner' in job.types:
# ss_name = job.types['inner']
# TODO: test if container is an AR job
slots = all_slot_sets[ss_name].slots
t_e = job.start_time + walltime - job_security_time
sid_left, sid_right = get_encompassing_slots(
slots, job.start_time, t_e)
if job.ts or (job.ph == ALLOW):
itvs_avail = intersec_ts_ph_itvs_slots(
slots, sid_left, sid_right, job)
else:
itvs_avail = intersec_itvs_slots(slots, sid_left, sid_right)
itvs = find_resource_hierarchies_job(
itvs_avail, hy_res_rqts, resource_set.hierarchy)
if ('QUOTAS' in config) and (config['QUOTAS'] == 'yes'):
nb_res = itvs_size(intersec(itvs, resource_set.default_resource_itvs))
res = check_slots_quotas(slots, sid_left, sid_right, job, nb_res, walltime)
(quotas_ok, quotas_msg, rule, value) = res
if not quotas_ok:
itvs = []
logger.info("Quotas limitaion reached, job:" + str(job.id) +
", " + quotas_msg + ", rule: " + str(rule) +
", value: " + str(value))
set_job_state(job.id, 'toError')
set_job_message(job.id,
"This advance reservation cannot run due to quotas")
if itvs == []:
# not enough resource available
logger.warn("[" + str(job.id) +
"] advance reservation cannot be validated, not enough resources")
set_job_state(job.id, 'toError')
set_job_message(job.id, "This advance reservation cannot run")
else:
# The reservation can be scheduled
logger.debug(
"[" + str(job.id) + "] advance reservation is validated")
job.moldable_id = moldable_id
job.res_set = itvs
ar_jobs_scheduled[job.id] = job
# if 'container' in job.types
# slot = Slot(1, 0, 0, job.res_set[:], job.start_time,
# job.start_time + job.walltime - job_security_time)
# slot.show()
# slots_sets[job.id] = SlotSet(slot)
set_job_state(job.id, 'toAckReservation')
set_job_resa_state(job.id, 'Scheduled')
if ar_jobs_scheduled != []:
logger.debug("Save AR jobs' assignements in database")
save_assigns(ar_jobs_scheduled, resource_set)
logger.debug("Queue " + queue_name + ": end processing of new reservations")
def handle_waiting_reservation_jobs(queue_name, resource_set, job_security_time, current_time_sec):
logger.debug("Queue " + queue_name +
": begin processing accepted Advance Reservations")
ar_jobs = get_waiting_scheduled_AR_jobs(queue_name, resource_set, job_security_time, current_time_sec)
for job in ar_jobs:
moldable_id = job.moldable_id
walltime = job.walltime
# Test if AR job is expired and handle it
if (current_time_sec > (job.start_time + walltime)):
logger.warn("[" + str(job.id) +
"] set job state to Error: avdance reservation expired and couldn't be started")
set_job_state(job.id, 'Error')
set_job_message(job.id, "Reservation expired and couldn't be started.")
else:
# Determine current available ressources
avail_res = intersec(resource_set.roid_itvs, job.res_set)
# Test if the AR job is waiting to be launched due to nodes' unavailabilities
if (avail_res == []) and (job.start_time < current_time_sec):
logger.warn("[%s] advance reservation is waiting because no resource is present"
% str(job.id))
# Delay launching time
set_gantt_job_start_time(moldable_id, current_time_sec + 1)
elif (job.start_time < current_time_sec):
if (job.start_time + reservation_waiting_timeout) > current_time_sec:
if not equal_itvs(avail_res, job.res_set):
# The expected ressources are not all available,
# wait the specified timeout
logger.warn("[" + str(job.id) +
"] advance reservation is waiting because not all \
resources are available yet")
set_gantt_job_start_time(moldable_id, current_time_sec + 1)
else:
# It's time to launch the AR job, remove missing ressources
missing_resources_itvs = sub_intervals(job.res_set, avail_res)
remove_gantt_resource_job(moldable_id, missing_resources_itvs,
resource_set)
logger.warn("[" + str(job.id) +
"remove some resources assigned to this advance reservation, \
because there are not Alive")
add_new_event('SCHEDULER_REDUCE_NB_RESSOURCES_FOR_ADVANCE_RESERVATION',
job.id,
"[MetaSched] Reduce the number of resources for the job "
+ str(job.id))
nb_res = itvs_size(job.res_set) - itvs_size(missing_resources_itvs)
new_message = re.sub(r'R=\d+', 'R=' + str(nb_res), job.message)
if new_message != job.message:
set_job_message(job.id, new_message)
logger.debug("Queue " + queue_name +
": end processing of reservations with missing resources")
