def set_moldable_job_max_time(moldable_id, walltime):
db.query(MoldableJobDescription)\
.filter(MoldableJobDescription.id == moldable_id)\
.update({MoldableJobDescription.walltime: walltime})
db.commit()
def set_gantt_job_start_time(moldable_id, current_time_sec):
db.query(GanttJobsPrediction)\
.filter(GanttJobsPrediction.moldable_id == moldable_id)\
.update({GanttJobsPrediction.start_time: current_time_sec})
db.commit()
def set_job_start_time_assigned_moldable_id(jid, start_time, moldable_id):
# db.query(Job).update({Job.start_time:
# start_time,Job.assigned_moldable_job: moldable_id}).filter(Job.id ==
# jid)
db.query(Job).filter(Job.id == jid).update(
{Job.start_time: start_time, Job.assigned_moldable_job: moldable_id})
db.commit()
def search_idle_nodes(date):
result = db.query(Resource.network_address)\
.filter(Resource.id == GanttJobsResource.resource_id)\
.filter(GanttJobsPrediction.start_time <= date)\
.filter(Resource.network_address != '')\
.filter(Resource.type == 'default')\
.filter(GanttJobsPrediction.moldable_id == GanttJobsResource.moldable_id)\
.group_by(Resource.network_address)\
.all()
busy_nodes = {}
for network_address in result:
if network_address not in busy_nodes:
busy_nodes[network_address] = True
result = db.query(Resource.network_address,
func.max(Resource.last_job_date))\
.filter(Resource.state == 'Alive')\
.filter(Resource.network_address != '')\
.filter(Resource.type == 'default')\
.filter(Resource.available_upto < 2147483647)\
.filter(Resource.available_upto > 0)\
.group_by(Resource.network_address)\
.all()
idle_nodes = {}
for x in result:
network_address, last_job_date = x
if network_address not in busy_nodes:
idle_nodes[network_address] = last_job_date
return idle_nodes
def call_external_scheduler(binpath, scheduled_jobs, all_slot_sets,
resource_set, job_security_time, queue,
initial_time_sec, initial_time_sql): # pragma: no cover
cmd_scheduler = binpath + "schedulers/" + queue.scheduler_policy
child_launched = True
# TODO TO CONFIRM
sched_exit_code = 0
sched_signal_num = 0
sched_dumped_core = 0
try:
child = Popen([cmd_scheduler, queue.name, str(
initial_time_sec), initial_time_sql], stdout=subprocess.PIPE)
for line in iter(child.stdout.readline, ''):
logger.debug("Read on the scheduler output:" + str(line.rstrip()))
# TODO SCHEDULER_LAUNCHER_OPTIMIZATION
# if
# ((get_conf_with_default_param('SCHEDULER_LAUNCHER_OPTIMIZATION',
# 'yes') eq 'yes') and
child.wait()
rc = child.returncode
sched_exit_code, sched_signal_num, sched_dumped_core = rc >> 8, rc & 0x7f, bool(
rc & 0x80)
except OSError as e:
child_launched = False
logger.warn(str(e) + " Cannot run: " + cmd_scheduler + " " + queue.name + " " +
str(initial_time_sec) + " " + initial_time_sql)
if (not child_launched) or (sched_signal_num != 0) or (sched_dumped_core != 0):
logger.error("Execution of " + queue.scheduler_policy +
" failed, inactivating queue " + queue.name + " (see `oarnotify')")
# stop queue
db.query(Queue).filter(Queue.name == queue.name).update({'state': 'notActive'})
if sched_exit_code != 0:
logger.error("Scheduler " + queue.scheduler_policy + " returned a bad value: " +
str(sched_exit_code) + ". Inactivating queue " + queue.scheduler_policy +
" (see `oarnotify')")
# stop queue
db.query(Queue).filter(Queue.name == queue.name).update({'state': 'notActive'})
# retrieve jobs and assignement decision from previous scheduling step
scheduled_jobs = get_after_sched_no_AR_jobs(queue.name, resource_set,
job_security_time, initial_time_sec)
if scheduled_jobs != []:
if queue.name == 'besteffort':
filter_besteffort = False
else:
filter_besteffort = True
set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs,
job_security_time, initial_time_sec,
filter_besteffort)
def set_job_resa_state(job_id, state):
''' sets the reservation field of the job of id passed in parameter
parameters : base, jobid, state
return value : None
side effects : changes the field state of the job in the table Jobs
'''
db.query(Job).filter(Job.id == job_id).update({Job.reservation: state})
db.commit()
def extra_metasched_foo(prev_queue, plt, scheduled_jobs, all_slot_sets,
job_security_time, queue, initial_time_sec,
extra_metasched_config):
if prev_queue is None:
# set first resource deployable
first_id = db.query(Resource).first().id
db.query(Resource).filter(Resource.id == first_id)\
.update({Resource.deploy: 'YES'}, synchronize_session=False)
db.commit()
def remove_gantt_resource_job(moldable_id, job_res_set, resource_set):
riods = itvs2ids(job_res_set)
resource_ids = [resource_set.rid_o2i[rid] for rid in riods]
db.query(GanttJobsResource)\
.filter(GanttJobsResource.moldable_id == moldable_id)\
.filter(~GanttJobsResource.resource_id.in_(tuple(resource_ids)))\
.delete(synchronize_session=False)
db.commit()
def update_gantt_visualization():
db.query(GanttJobsPredictionsVisu).delete()
db.query(GanttJobsResourcesVisu).delete()
db.commit()
sql_queries = ["INSERT INTO gantt_jobs_predictions_visu SELECT * FROM gantt_jobs_predictions",
"INSERT INTO gantt_jobs_resources_visu SELECT * FROM gantt_jobs_resources"
]
for query in sql_queries:
db.session.execute(query)
db.commit()
def update_current_scheduler_priority(job, value, state):
"""Update the scheduler_priority field of the table resources
"""
# TO FINISH
# TODO: MOVE TO resource.py ???
logger.info("update_current_scheduler_priority " +
" job.id: " + str(job.id) + ", state: " + state + ", value: "
+ str(value))
if "SCHEDULER_PRIORITY_HIERARCHY_ORDER" in config:
sched_priority = config["SCHEDULER_PRIORITY_HIERARCHY_ORDER"]
if ((('besteffort' in job.types) or ('timesharing' in job.types)) and
(((state == 'START') and
is_an_event_exists(job.id, "SCHEDULER_PRIORITY_UPDATED_START") <= 0) or
((state == 'STOP') and is_an_event_exists(job.id, "SCHEDULER_PRIORITY_UPDATED_START") > 0))):
coeff = 1
if ('besteffort' in job.types) and (not ('timesharing' in job.types)):
coeff = 10
index = 0
for f in sched_priority.split('/'):
if f == '':
continue
elif f == 'resource_id':
f = 'id'
index += 1
res = db.query(distinct(getattr(Resource, f)))\
.filter(AssignedResource.index == 'CURRENT')\
.filter(AssignedResource.moldable_id == job.assigned_moldable_job)\
.filter(AssignedResource.resource_id == Resource.id)\
.all()
resources = tuple(r[0] for r in res)
if resources == ():
return
incr_priority = int(value) * index * coeff
db.query(Resource)\
.filter((getattr(Resource, f)).in_(resources))\
.update({Resource.scheduler_priority: incr_priority}, synchronize_session=False)
add_new_event('SCHEDULER_PRIORITY_UPDATED_' + state, job.id,
'Scheduler priority for job ' + str(job.id) +
'updated (' + sched_priority + ')')
def update_scheduler_last_job_date(date, moldable_id):
''' used to allow search_idle_nodes to operate for dynamic node management feature (Hulot)
'''
if db.dialect == "sqlite":
subquery = db.query(AssignedResource.resource_id).filter_by(moldable_id=moldable_id)\
.subquery()
db.query(Resource).filter(Resource.id.in_(subquery))\
.update({Resource.last_job_date: date}, synchronize_session=False)
else:
db.query(Resource).filter(AssignedResource.moldable_id == moldable_id)\
.filter(Resource.id == AssignedResource.resource_id)\
.update({Resource.last_job_date: date}, synchronize_session=False)
db.commit()
def test_db_all_in_one_wakeup_node_energy_saving_internal_1(monkeypatch):
config['ENERGY_SAVING_INTERNAL'] = 'yes'
insert_job(res=[(60, [('resource_id=4', "")])], properties="")
now = get_date()
# Suspend nodes
db.query(Resource).update({Resource.state: 'Absent', Resource.available_upto: now + 1000},
synchronize_session=False)
db.commit()
meta_schedule('internal')
job = db['Job'].query.one()
print(job.state)
print(node_list)
assert (job.state == 'Waiting')
def get_gantt_jobs_to_launch(resource_set, job_security_time, now):
# get unlaunchable jobs
# NOT USED launcher will manage these cases ??? (MUST BE CONFIRMED)
#
#result = db.query(distinct(Job.id))\
# .filter(GanttJobsPrediction.start_time <= now)\
# .filter(Job.state == "Waiting")\
# .filter(Job.id == MoldableJobDescription.job_id)\
# .filter(MoldableJobDescription.id == GanttJobsPrediction.moldable_id)\
# .filter(GanttJobsResource.moldable_id == GanttJobsPrediction.moldable_id)\
# AND (resources.state IN (\'Dead\',\'Suspected\',\'Absent\')
# OR resources.next_state IN (\'Dead\',\'Suspected\',\'Absent\'))
#
# .all()
result = db.query(Job,
GanttJobsPrediction.moldable_id,
GanttJobsPrediction.start_time,
MoldableJobDescription.walltime,
GanttJobsResource.resource_id)\
.filter(GanttJobsPrediction.start_time <= now)\
.filter(Job.state == "Waiting")\
.filter(Job.id == MoldableJobDescription.job_id)\
.filter(MoldableJobDescription.id == GanttJobsPrediction.moldable_id)\
.filter(GanttJobsResource.moldable_id
== GanttJobsPrediction.moldable_id)\
.filter(Resource.id == GanttJobsResource.resource_id)\
.filter(Resource.state == 'Alive')\
.all()
jobs, jobs_lst, _, rid2jid = extract_scheduled_jobs(result, resource_set,
job_security_time, now)
return (jobs, jobs_lst, rid2jid)
def resume_job_action(job_id):
'''resume_job_action performs all action when a job is suspended'''
set_job_state(job_id, 'Running')
resources = get_current_resources_with_suspended_job()
if resources != ():
db.query(Resource)\
.filter(~Resource.id.in_(resources))\
.update({Resource.suspended_jobs: 'NO'}, synchronize_session=False)
else:
db.query(Resource)\
.update({Resource.suspended_jobs: 'NO'}, synchronize_session=False)
db.commit()
def get_cpuset_values(cpuset_field, moldable_id):
# TODO TOFINISH
logger.warning("get_cpuset_values is NOT ENTIRELY IMPLEMENTED")
sql_where_string = "\'0\'"
if "SCHEDULER_RESOURCES_ALWAYS_ASSIGNED_TYPE" in config:
resources_to_always_add_type = config["SCHEDULER_RESOURCES_ALWAYS_ASSIGNED_TYPE"]
else:
resources_to_always_add_type = ""
# TODO
if resources_to_always_add_type != "":
sql_where_string = "resources.type = \'$resources_to_always_add_type\'"
results = db.query(Resource)\
.filter(AssignedResource.moldable_id == moldable_id)\
.filter(AssignedResource.resource_id == Resource.id)\
# my $sth = $dbh->prepare(" SELECT resources.network_address, resources.$cpuset_field
# FROM resources, assigned_resources
# WHERE
# assigned_resources.moldable_job_id = $moldable_job_id AND
# assigned_resources.resource_id = resources.resource_id AND
# resources.network_address != \'\' AND
# (resources.type = \'default\' OR
# $sql_where_string)
# GROUP BY resources.network_address, resources.$cpuset_field
# ");
return results
def frag_job(jid):
if 'OARDO_USER' in os.environ:
luser = os.environ['OARDO_USER']
else:
luser = os.environ['USER']
job = get_job(jid)
if (job is not None) and ((luser == job.user)
or (luser == 'oar')
or (luser == 'root')):
res = db.query(FragJob).filter(FragJob.job_id == jid).all()
if len(res) == 0:
date = tools.get_date()
frajob = FragJob(job_id=jid, date=date)
db.add(frajob)
db.commit()
add_new_event("FRAG_JOB_REQUEST",
jid, "User %s requested to frag the job %s"
% (luser, str(jid)))
return 0
else:
# Job already killed
return -2
else:
return -1
def test_db_all_in_one_wakeup_node_1(monkeypatch):
insert_job(res=[(60, [('resource_id=4', "")])], properties="")
now = get_date()
# Suspend nodes
db.query(Resource).update({Resource.state: 'Absent', Resource.available_upto: now + 1000},
synchronize_session=False)
db.commit()
meta_schedule('internal')
job = db['Job'].query.one()
print(job.state)
print(node_list)
assert (job.state == 'Waiting')
assert (node_list == [u'localhost0', u'localhost1'])
def get_current_resources_with_suspended_job():
res = db.query(AssignedResource.resource_id).filter(AssignedResource.index == 'CURRENT')\
.filter(Job.state == 'Suspended')\
.filter(Job.assigned_moldable_job == AssignedResource.moldable_id)\
.all()
return tuple(r for r in res)
def test_db_all_in_one_AR_4(monkeypatch):
now = get_date()
job = insert_and_sched_ar(now + 10)
new_start_time = now - 20
db.query(GanttJobsPrediction).update({GanttJobsPrediction.start_time: new_start_time},
synchronize_session=False)
db.commit()
meta_schedule('internal')
job = db['Job'].query.one()
print('\n', job.id, job.state, ' ', job.reservation, job.start_time)
assert job.state == 'toLaunch'
def get_next_job_date_on_node(hostname):
result = db.query(func.min(GanttJobsPrediction.start_time))\
.filter(Resource.network_address == hostname)\
.filter(GanttJobsResource.resource_id == Resource.id)\
.filter(GanttJobsPrediction.moldable_id == GanttJobsResource.moldable_id)\
.scalar()
return result
def get_current_not_waiting_jobs():
jobs = db.query(Job).filter(Job.state != "Waiting").all()
jobs_by_state = {}
for job in jobs:
if job.state not in jobs_by_state:
jobs_by_state[job.state] = []
jobs_by_state[job.state].append(job)
return (jobs_by_state)
def get_job(job_id): # pragma: no cover
try:
job = db.query(Job).filter(Job.id == job_id).one()
except Exception as e:
logger.warning("get_job(" + str(job_id) + ") raises execption: " + str(e))
return None
else:
return job
def get_last_wake_up_date_of_node(hostname):
result = db.query(EventLog.date)\
.filter(EventLogHostname.event_id == EventLog.id)\
.filter(EventLogHostname.hostname == hostname)\
.filter(EventLog.type == 'WAKEUP_NODE')\
.order_by(EventLog.date.desc()).limit(1).scalar()
return result
def test_db_all_in_one_AR_6(monkeypatch):
now = get_date()
job = insert_and_sched_ar(now + 10, 600)
new_start_time = now - 350
set_jobs_start_time(tuple([job.id]), new_start_time)
db.query(GanttJobsPrediction).update({GanttJobsPrediction.start_time: new_start_time},
synchronize_session=False)
# db.query(Resource).update({Resource.state: 'Suspected'}, synchronize_session=False)
meta_schedule('internal')
job = db['Job'].query.one()
print('\n', job.id, job.state, ' ', job.reservation, job.start_time)
assert job.state == 'Waiting'
def test_db_all_in_one_sleep_node_1(monkeypatch):
now = get_date()
insert_job(res=[(60, [('resource_id=1', "")])], properties="")
# Suspend nodes
# pdb.set_trace()
db.query(Resource).update({Resource.available_upto: now + 50000},
synchronize_session=False)
db.commit()
meta_schedule('internal')
job = db['Job'].query.one()
print(job.state)
print(node_list)
assert (job.state == 'toLaunch')
assert (node_list == [u'localhost2', u'localhost1'] or
node_list == [u'localhost1', u'localhost2'])
def add_resource_job_pairs(moldable_id):
resources_mld_ids = db.query(GanttJobsResource)\
.filter(GanttJobsResource.moldable_id == moldable_id)\
.all()
assigned_resources = [{'moldable_job_id': res_mld_id.moldable_id,
'resource_id': res_mld_id.resource_id} for res_mld_id in resources_mld_ids]
db.session.execute(AssignedResource.__table__.insert(), assigned_resources)
db.commit()
def add_resource_jobs_pairs(tuple_mld_ids): # pragma: no cover
resources_mld_ids = db.query(GanttJobsResource)\
.filter(GanttJobsResource.job_id.in_(tuple_mld_ids))\
.all()
assigned_resources = [{'moldable_job_id': res_mld_id.moldable_id,
'resource_id': res_mld_id.resource_id} for res_mld_id in resources_mld_ids]
db.session.execute(AssignedResource.__table__.insert(), assigned_resources)
db.commit()
def scheduleJobs(self):
print("Sheduling Round")
real_time = time.time()
if self.platform_model == "simu":
schedule_cycle(self.platform, self.env.now, "default")
# retrieve jobs to launch
jids_to_launch = []
for jid, job in iteritems(self.platform.assigned_jobs):
print("job.start_time %s" % job.start_time)
if (job.start_time == self.env.now) and (job.state == "Waiting"):
self.waiting_jids.remove(jid)
jids_to_launch.append(jid)
job.state = "Running"
print("tolaunch: %s" % jid)
self.platform.running_jids.append(jid)
else:
print("call meta_schedule('internal')")
meta_schedule("internal", plt)
result = db.query(Job).filter(Job.state == "toLaunch").order_by(Job.id).all()
for job_db in result:
set_job_state(job_db.id, "Running")
jid = self.db_jid2s_jid[job_db.id]
self.waiting_jids.remove(jid)
jids_to_launch.append(jid)
self.jobs[jid].state = "Running"
print("_tolaunch: %s" % jid)
self.platform.running_jids.append(jid)
print("Ids of jobs to launch: ", *jids_to_launch)
print("Time befort scheduling round: ", self.bs._current_time, self.sched_delay)
# update time
real_sched_time = time.time() - real_time
if self.sched_delay == -1:
self.bs.consume_time(real_sched_time) # TODO
else:
self.bs.consume_time(self.sched_delay)
self.env.now = self.bs._current_time
print("Time after scheduling round: ", self.bs._current_time)
# send to uds
if len(jids_to_launch) > 0:
scheduled_jobs = []
jobs_res = {}
for jid in jids_to_launch:
ds_job = self.jobs[jid].ds_job
res = itvs2batsim_str0(self.jobs[jid].res_set)
scheduled_jobs.append(ds_job)
jobs_res[ds_job.id] = res
self.bs.start_jobs(scheduled_jobs, jobs_res)
def get_gantt_waiting_interactive_prediction_date():
req = db.query(Job.id,
Job.info_type,
GanttJobsPrediction.start_time,
Job.message)\
.filter(Job.state == 'Waiting')\
.filter(Job.type == 'INTERACTIVE')\
.filter(Job.reservation == 'None')\
.filter(MoldableJobDescription.job_id == Job.id)\
.filter(GanttJobsPrediction.moldable_id == MoldableJobDescription.id)\
.all()
return req
def get_waiting_reservation_jobs_specific_queue(queue_name):
'''Get all waiting reservation jobs in the specified queue
parameter : database ref, queuename
return an array of job informations
'''
waiting_scheduled_ar_jobs = db.query(Job)\
.filter((Job.state == 'Waiting')
|
(Job.state == 'toAckReservation'))\
.filter(Job.reservation == 'Scheduled')\
.filter(Job.queue_name == queue_name)\
.order_by(Job.id).all()
return waiting_scheduled_ar_jobs
