def bench_job_same(nb_job_exp=10, job_size=100, save_assign="default"):
delete_tables()
print("# bench: ", save_assign)
print("# nb_j, job_size, time")
for j in range(nb_job_exp):
nb_j = 2 ** j
jobs = generate_jobs(nb_j, job_size)
start = time.time()
# getattr(oar.kao.job, "save_assigns_bulk")(jobs, rs)
if save_assign == "bulk_0":
save_assigns_bulk_0(jobs, rs)
elif save_assign == "redis_0":
save_assigns_redis_0(jobs, rs)
else:
save_assigns(jobs, rs)
end = time.time()
print(nb_j, job_size, end - start)
delete_tables()
def bench_job_same(nb_job_exp=10, job_size=100, save_assign='default'):
delete_tables()
print('# bench: ', save_assign)
print('# nb_j, job_size, time')
for j in range(nb_job_exp):
nb_j = 2**j
jobs = generate_jobs(nb_j, job_size)
start = time.time()
# getattr(oar.kao.job, "save_assigns_bulk")(jobs, rs)
if save_assign == 'bulk_0':
save_assigns_bulk_0(jobs, rs)
elif save_assign == 'redis_0':
save_assigns_redis_0(jobs, rs)
elif save_assign == 'redis_pipeline_0':
save_assigns_redis_pipeline_0(jobs, rs)
elif save_assign == 'redis_pipeline_1':
save_assigns_redis_pipeline_1(jobs, rs)
else:
save_assigns(jobs, rs)
end = time.time()
print(nb_j, job_size, end - start)
delete_tables()
def save_assigns_simu_and_default(self, jobs, resource_set):
print("save_assigns_simu_and_default........................")
# assigned_jobs = {}
for jid, job in iteritems(jobs):
sid = self.db_jid2s_jid[jid]
jobsimu = self.jobs[sid]
jres_set = job.res_set
r_ids = [resource_set.rid_o2i[roid] for roid in itvs2ids(jres_set)]
jobsimu.res_set = unordered_ids2itvs(r_ids)
print("save assign jid, sid, res_set: ", jid, " ", sid, " ", jobsimu.res_set)
jobsimu.start_time = job.start_time
jobsimu.walltime = job.walltime
# assigned_jobs[sid] = jobsimu
# self.assigned_jobs = assigned_jobs
return save_assigns(jobs, resource_set)
def check_reservation_jobs(plt, resource_set, queue_name, all_slot_sets, current_time_sec):
"""Processing of new Advance Reservations"""
logger.debug("Queue " + queue_name + ": begin processing of new reservations")
ar_jobs_scheduled = {}
ar_jobs, ar_jids, nb_ar_jobs = plt.get_waiting_jobs(
queue_name, 'toSchedule')
logger.debug("nb_ar_jobs:" + str(nb_ar_jobs))
if nb_ar_jobs > 0:
job_security_time = int(config['SCHEDULER_JOB_SECURITY_TIME'])
plt.get_data_jobs(ar_jobs, ar_jids, resource_set, job_security_time)
logger.debug("Try and schedule new reservations")
for jid in ar_jids:
job = ar_jobs[jid]
logger.debug(
"Find resource for Advance Reservation job:" + str(job.id))
# It is a reservation, we take care only of the first moldable job
moldable_id, walltime, hy_res_rqts = job.mld_res_rqts[0]
# test if reservation is too old
if current_time_sec >= (job.start_time + walltime):
logger.warn(
"[" + str(job.id) + "] Canceling job: reservation is too old")
set_job_message(job.id, "Reservation too old")
set_job_state(job.id, 'toError')
continue
else:
if job.start_time < current_time_sec:
# TODO update to DB ????
job.start_time = current_time_sec
ss_name = 'default'
# TODO container
# if 'inner' in job.types:
# ss_name = job.types['inner']
# TODO: test if container is an AR job
slots = all_slot_sets[ss_name].slots
t_e = job.start_time + walltime - job_security_time
sid_left, sid_right = get_encompassing_slots(
slots, job.start_time, t_e)
if job.ts or (job.ph == ALLOW):
itvs_avail = intersec_ts_ph_itvs_slots(
slots, sid_left, sid_right, job)
else:
itvs_avail = intersec_itvs_slots(slots, sid_left, sid_right)
itvs = find_resource_hierarchies_job(
itvs_avail, hy_res_rqts, resource_set.hierarchy)
if ('QUOTAS' in config) and (config['QUOTAS'] == 'yes'):
nb_res = itvs_size(intersec(itvs, resource_set.default_resource_itvs))
res = check_slots_quotas(slots, sid_left, sid_right, job, nb_res, walltime)
(quotas_ok, quotas_msg, rule, value) = res
if not quotas_ok:
itvs = []
logger.info("Quotas limitaion reached, job:" + str(job.id) +
", " + quotas_msg + ", rule: " + str(rule) +
", value: " + str(value))
set_job_state(job.id, 'toError')
set_job_message(job.id,
"This advance reservation cannot run due to quotas")
if itvs == []:
# not enough resource available
logger.warn("[" + str(job.id) +
"] advance reservation cannot be validated, not enough resources")
set_job_state(job.id, 'toError')
set_job_message(job.id, "This advance reservation cannot run")
else:
# The reservation can be scheduled
logger.debug(
"[" + str(job.id) + "] advance reservation is validated")
job.moldable_id = moldable_id
job.res_set = itvs
ar_jobs_scheduled[job.id] = job
# if 'container' in job.types
# slot = Slot(1, 0, 0, job.res_set[:], job.start_time,
# job.start_time + job.walltime - job_security_time)
# slot.show()
# slots_sets[job.id] = SlotSet(slot)
set_job_state(job.id, 'toAckReservation')
set_job_resa_state(job.id, 'Scheduled')
if ar_jobs_scheduled != []:
logger.debug("Save AR jobs' assignements in database")
save_assigns(ar_jobs_scheduled, resource_set)
logger.debug("Queue " + queue_name + ": end processing of new reservations")
def save_assigns_default(self, *args):
return save_assigns(*args)