def scheduleJobs(self): print("Sheduling Round") real_time = time.time() if self.platform_model == "simu": schedule_cycle(self.platform, self.env.now, "default") # retrieve jobs to launch jids_to_launch = [] for jid, job in iteritems(self.platform.assigned_jobs): print("job.start_time %s" % job.start_time) if (job.start_time == self.env.now) and (job.state == "Waiting"): self.waiting_jids.remove(jid) jids_to_launch.append(jid) job.state = "Running" print("tolaunch: %s" % jid) self.platform.running_jids.append(jid) else: print("call meta_schedule('internal')") meta_schedule("internal", plt) result = db.query(Job).filter(Job.state == "toLaunch").order_by(Job.id).all() for job_db in result: set_job_state(job_db.id, "Running") jid = self.db_jid2s_jid[job_db.id] self.waiting_jids.remove(jid) jids_to_launch.append(jid) self.jobs[jid].state = "Running" print("_tolaunch: %s" % jid) self.platform.running_jids.append(jid) print("Ids of jobs to launch: ", *jids_to_launch) print("Time befort scheduling round: ", self.bs._current_time, self.sched_delay) # update time real_sched_time = time.time() - real_time if self.sched_delay == -1: self.bs.consume_time(real_sched_time) # TODO else: self.bs.consume_time(self.sched_delay) self.env.now = self.bs._current_time print("Time after scheduling round: ", self.bs._current_time) # send to uds if len(jids_to_launch) > 0: scheduled_jobs = [] jobs_res = {} for jid in jids_to_launch: ds_job = self.jobs[jid].ds_job res = itvs2batsim_str0(self.jobs[jid].res_set) scheduled_jobs.append(ds_job) jobs_res[ds_job.id] = res self.bs.start_jobs(scheduled_jobs, jobs_res)
def test_suspend_resume_1(monkeypatch): # now = get_date() insert_job(res=[(60, [('resource_id=4', "")])], properties="") meta_schedule('internal') job = db['Job'].query.one() print(job.state) set_job_state(job.id, 'Resuming') job = db['Job'].query.one() print(job.state) meta_schedule('internal') assert(job.state == 'Resuming')
def test_suspend_resume_2(monkeypatch): config['JUST_BEFORE_RESUME_EXEC_FILE'] = 'sleep 2' # now = get_date() insert_job(res=[(60, [('resource_id=4', "")])], properties="") meta_schedule('internal') job = db['Job'].query.one() print(job.state) set_job_state(job.id, 'Resuming') job = db['Job'].query.one() print(job.state) meta_schedule('internal') assert(job.state == 'Resuming')
def onJobCompletion(self, data_storage_job): data_storage_jobs = [data_storage_job] # TODO vectorize in batsim.py !!! for job in data_storage_jobs: jid = self.ujid_l.index(job.id) self.jobs_completed.append(jid) if jid in self.platform.running_jids: self.platform.running_jids.remove(jid) if self.platform_model == "batsim-db": set_job_state(self.jobs[jid].db_jid, "Terminated") self.scheduleJobs()
def test_db_all_in_one_BE_2(monkeypatch): # TODO TOFINISH db['Queue'].create(name='besteffort', priority=3, scheduler_policy='kamelot', state='Active') insert_job(res=[(100, [('resource_id=1', "")])], queue_name='besteffort', types=['besteffort', "timesharing=*,*"]) meta_schedule('internal') job = db['Job'].query.one() set_job_state(job.id, 'Running') insert_job(res=[(50, [('resource_id=1', "")])], types=["timesharing=*,*"]) meta_schedule('internal') jobs = db['Job'].query.all() print(jobs[1].id, jobs[1].state) # assert (jobs[1].state == 'toLaunch') assert (jobs[1].state == 'Waiting')
def prepare_job_to_be_launched(job, current_time_sec): # TODO ??? # my $running_date = $current_time_sec; # if ($running_date < $job_submission_time){ # $running_date = $job_submission_time; # } # OAR::IO::set_running_date_arbitrary($base, $job_id, $running_date); # OAR::IO::set_assigned_moldable_job($base, $job_id, $moldable_job_id); # set start_time an for jobs to launch set_job_start_time_assigned_moldable_id(job.id, current_time_sec, job.moldable_id) # fix resource assignement add_resource_job_pairs(job.moldable_id) set_job_state(job.id, 'toLaunch') notify_to_run_job(job.id)
def sched_loop(self): nb_completed_jobs = 0 while nb_completed_jobs < self.nb_jobs: now_float, jobs_submitted, new_jobs_completed = read_bat_msg(self.sock) # now_str = "10" # jobs_submitted = [1] # new_jobs_completed = [] if jobs_submitted: for jid in jobs_submitted: self.waiting_jids.add(jid) if self.platform_model == "batsim-db": print('set_job_state("Waiting"):', self.jobs[jid].db_jid) set_job_state(self.jobs[jid].db_jid, "Waiting") nb_completed_jobs += len(new_jobs_completed) print("new job completed: %s" % new_jobs_completed) for jid in new_jobs_completed: jobs_completed.append(jid) if jid in self.platform.running_jids: self.platform.running_jids.remove(jid) if self.platform_model == "batsim-db": set_job_state(self.jobs[jid].db_jid, "Terminated") now = int(now_float) self.env.now = now # TODO can be remove ??? real_time = time.time() print("jobs running: %s" % self.platform.running_jids) print("jobs waiting: %s" % self.waiting_jids) print("jobs completed: %s" % jobs_completed) jids_to_launch = [] if self.platform_model == "simu": print("call schedule_cycle.... %s" % now) schedule_cycle(self.platform, now, "default") # retrieve jobs to launch jids_to_launch = [] for jid, job in iteritems(self.platform.assigned_jobs): print("job.start_time %s" % job.start_time) if (job.start_time == now) and (job.state == "Waiting"): self.waiting_jids.remove(jid) jids_to_launch.append(jid) job.state = "Running" print("tolaunch: %s" % jid) self.platform.running_jids.append(jid) else: print("call meta_schedule('internal')") meta_schedule("internal", plt) # Launching phase # Retrieve job to Launch result = db.query(Job).filter(Job.state == "toLaunch").order_by(Job.id).all() for job_db in result: set_job_state(job_db.id, "Running") jid = self.db_jid2s_jid[job_db.id] self.waiting_jids.remove(jid) jids_to_launch.append(jid) self.jobs[jid].state = "Running" print("_tolaunch: %s" % jid) self.platform.running_jids.append(jid) real_sched_time = time.time() - real_time if self.sched_delay == -1: now_float += real_sched_time else: now_float += self.sched_delay send_bat_msg(self.sock, now_float, jids_to_launch, self.jobs)
def meta_schedule(mode='internal', plt=Platform()): exit_code = 0 job_security_time = int(config['SCHEDULER_JOB_SECURITY_TIME']) if ('QUOTAS' in config) and (config['QUOTAS'] == 'yes'): if 'QUOTAS_FILE' not in config: config['QUOTAS_FILE'] = './quotas_conf.json' load_quotas_rules() tools.init_judas_notify_user() tools.create_almighty_socket() logger.debug( "Retrieve information for already scheduled reservations from \ database before flush (keep assign resources)") # reservation ??. initial_time_sec = tools.get_date() # time.time() initial_time_sql = local_to_sql(initial_time_sec) current_time_sec = initial_time_sec current_time_sql = initial_time_sql gantt_init_results = gantt_init_with_running_jobs(plt, initial_time_sec, job_security_time) all_slot_sets, scheduled_jobs, besteffort_rid2jid = gantt_init_results resource_set = plt.resource_set() # Path for user of external schedulers if 'OARDIR' in os.environ: binpath = os.environ['OARDIR'] + '/' else: binpath = '/usr/local/lib/oar/' logger.warning( "OARDIR env variable must be defined, " + binpath + " is used by default") for queue in db.query(Queue).order_by(text('priority DESC')).all(): if queue.state == 'Active': logger.debug("Queue " + queue.name + ": Launching scheduler " + queue.scheduler_policy + " at time " + initial_time_sql) if mode == 'external': # pragma: no cover call_external_scheduler(binpath, scheduled_jobs, all_slot_sets, resource_set, job_security_time, queue, initial_time_sec, initial_time_sql) else: call_internal_scheduler(plt, scheduled_jobs, all_slot_sets, job_security_time, queue, initial_time_sec) handle_waiting_reservation_jobs(queue.name, resource_set, job_security_time, current_time_sec) # handle_new_AR_jobs check_reservation_jobs( plt, resource_set, queue.name, all_slot_sets, current_time_sec) jobs_to_launch, jobs_to_launch_lst, rid2jid_to_launch = get_gantt_jobs_to_launch(resource_set, job_security_time, current_time_sec) if check_besteffort_jobs_to_kill(jobs_to_launch, rid2jid_to_launch, current_time_sec, besteffort_rid2jid, resource_set) == 1: # We must kill some besteffort jobs tools.notify_almighty('ChState') exit_code = 2 elif handle_jobs_to_launch(jobs_to_launch_lst, current_time_sec, current_time_sql) == 1: exit_code = 0 # Update visu gantt tables update_gantt_visualization() # Manage dynamic node feature flag_hulot = False timeout_cmd = int(config['SCHEDULER_TIMEOUT']) if ((('SCHEDULER_NODE_MANAGER_SLEEP_CMD' in config) or ((config['ENERGY_SAVING_INTERNAL'] == 'yes') and ('ENERGY_SAVING_NODE_MANAGER_SLEEP_CMD' in config))) and (('SCHEDULER_NODE_MANAGER_SLEEP_TIME' in config) and ('SCHEDULER_NODE_MANAGER_IDLE_TIME' in config))): # Look at nodes that are unused for a duration idle_duration = int(config['SCHEDULER_NODE_MANAGER_IDLE_TIME']) sleep_duration = int(config['SCHEDULER_NODE_MANAGER_SLEEP_TIME']) idle_nodes = search_idle_nodes(current_time_sec) tmp_time = current_time_sec - idle_duration node_halt = [] for node, idle_duration in iteritems(idle_nodes): if idle_duration < tmp_time: # Search if the node has enough time to sleep tmp = get_next_job_date_on_node(node) if (tmp is None) or (tmp - sleep_duration > current_time_sec): # Search if node has not been woken up recently wakeup_date = get_last_wake_up_date_of_node(node) if (wakeup_date is None) or (wakeup_date < tmp_time): node_halt.append(node) if node_halt != []: logger.debug("Powering off some nodes (energy saving): " + str(node_halt)) # Using the built-in energy saving module to shut down nodes if config['ENERGY_SAVING_INTERNAL'] == 'yes': if kao_tools.send_to_hulot('HALT', ' '.join(node_halt)): logger.error("Communication problem with the energy saving module (Hulot)\n") flag_hulot = 1 else: # Not using the built-in energy saving module to shut down nodes cmd = config['SCHEDULER_NODE_MANAGER_SLEEP_CMD'] if kao_tools.fork_and_feed_stdin(cmd, timeout_cmd, node_halt): logger.error("Command " + cmd + "timeouted (" + str(timeout_cmd) + "s) while trying to poweroff some nodes") if (('SCHEDULER_NODE_MANAGER_SLEEP_CMD' in config) or ((config['ENERGY_SAVING_INTERNAL'] == 'yes') and ('ENERGY_SAVING_NODE_MANAGER_SLEEP_CMD' in config))): # Get nodes which the scheduler wants to schedule jobs to, # but which are in the Absent state, to wake them up wakeup_time = int(config['SCHEDULER_NODE_MANAGER_WAKEUP_TIME']) nodes = get_gantt_hostname_to_wake_up(current_time_sec, wakeup_time) if nodes != []: logger.debug("Awaking some nodes: " + str(nodes)) # Using the built-in energy saving module to wake up nodes if config['ENERGY_SAVING_INTERNAL'] == 'yes': if kao_tools.send_to_hulot('WAKEUP', ' '.join(nodes)): logger.error("Communication problem with the energy saving module (Hulot)") flag_hulot = 1 else: # Not using the built-in energy saving module to wake up nodes cmd = config['SCHEDULER_NODE_MANAGER_WAKE_UP_CMD'] if kao_tools.fork_and_feed_stdin(cmd, timeout_cmd, nodes): logger.error("Command " + cmd + "timeouted (" + str(timeout_cmd) + "s) while trying to wake-up some nodes ") # Send CHECK signal to Hulot if needed if not flag_hulot and (config['ENERGY_SAVING_INTERNAL'] == 'yes'): if kao_tools.send_to_hulot('CHECK', []): logger.error("Communication problem with the energy saving module (Hulot)") # Retrieve jobs according to their state and excluding job in 'Waiting' state. jobs_by_state = get_current_not_waiting_jobs() # # Search jobs to resume # # # TODO: TOFINISH # if 'Resuming' in jobs_by_state: logger.warn("Resuming job is NOT ENTIRELY IMPLEMENTED") for job in jobs_by_state['Resuming']: other_jobs = get_jobs_on_resuming_job_resources(job.id) # TODO : look for timesharing other jobs. What do we do????? if other_jobs == []: # We can resume the job logger.debug("[" + str(job.id) + "] Resuming job") if 'noop' in job.types: resume_job_action(job.id) logger.debug("[" + str(job.id) + "] Resume NOOP job OK") else: script = config['JUST_BEFORE_RESUME_EXEC_FILE'] timeout = int(config['SUSPEND_RESUME_SCRIPT_TIMEOUT']) if timeout is None: timeout = kao_tools.get_default_suspend_resume_script_timeout() skip = 0 logger.debug("[" + str(job.id) + "] Running post suspend script: `" + script + " " + str(job.id) + "'") cmd_str = script + str(job.id) return_code = -1 try: return_code = call(cmd_str, shell=True, timeout=timeout) except TimeoutExpired as e: logger.error(str(e) + "[" + str(job.id) + "] Suspend script timeouted") add_new_event('RESUME_SCRIPT_ERROR', job.id, "Suspend script timeouted") if return_code != 0: str_error = "[" + str(job.id) + "] Suspend script error, return code = "\ + str(return_code) logger.error(str_error) add_new_event('RESUME_SCRIPT_ERROR', job.id, str_error) frag_job(job.id) tools.notify_almighty('Qdel') skip = 1 cpuset_nodes = None if 'JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD' in config: cpuset_field = config['JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD'] else: cpuset_field = "" if cpuset_field and (skip == 0): # TODO cpuset_name = job.user + "_" + str(job.id) cpuset_nodes = get_cpuset_values(cpuset_field, job.assigned_moldable_id) # TODO suspend_data_hash = {'name': cpuset_name, 'job_id': job.id, 'oarexec_pid_file': kao_tools.get_oar_pid_file_name(job.id)} if cpuset_nodes: # TODO taktuk_cmd = config['TAKTUK_CMD'] if 'SUSPEND_RESUME_FILE' in config: suspend_file = config['SUSPEND_RESUME_FILE'] else: # TODO suspend_file = kao_tools.get_default_suspend_resume_file() # # TODO: TOFINISH # # Notify oarsub -I when they will be launched for j_info in get_gantt_waiting_interactive_prediction_date(): job_id, job_info_type, job_start_time, job_message = j_info addr, port = job_info_type.split(':') new_start_prediction = local_to_sql(job_start_time) logger.debug("[" + str(job_id) + "] Notifying user of the start prediction: " + new_start_prediction + "(" + job_message + ")") tools.notify_tcp_socket(addr, port, "[" + initial_time_sql + "] Start prediction: " + new_start_prediction + " (" + job_message + ")") # Run the decisions # Process "toError" jobs if 'toError' in jobs_by_state: for job in jobs_by_state['toError']: addr, port = job.info_type.split(':') if job.type == 'INTERACTIVE' or\ (job.type == 'PASSIVE' and job.reservation == 'Scheduled'): logger.debug("Notify oarsub job (num:" + str(job.id) + ") in error; jobInfo=" + job.info_type) nb_sent1 = tools.notify_tcp_socket(addr, port, job.message + '\n') nb_sent2 = tools.notify_tcp_socket(addr, port, 'BAD JOB' + '\n') if (nb_sent1 == 0) or (nb_sent2 == 0): logger.warn( "Cannot open connection to oarsub client for" + str(job.id)) logger.debug("Set job " + str(job.id) + " to state Error") set_job_state(job.id, 'Error') # Process toAckReservation jobs if 'toAckReservation' in jobs_by_state: for job in jobs_by_state['toAckReservation']: addr, port = job.info_type.split(':') logger.debug( "Treate job" + str(job.id) + " in toAckReservation state") nb_sent = tools.notify_tcp_socket(addr, port, 'GOOD RESERVATION' + '\n') if nb_sent == 0: logger.warn( "Frag job " + str(job.id) + ", I cannot notify oarsub for the reservation") add_new_event('CANNOT_NOTIFY_OARSUB', str( job.id), "Can not notify oarsub for the job " + str(job.id)) # TODO ??? # OAR::IO::lock_table / OAR::IO::unlock_table($base) frag_job(job.id) exit_code = 2 else: logger.debug("Notify oarsub for a RESERVATION (idJob=" + str(job.id) + ") --> OK; jobInfo=" + job.info_type) set_job_state(job.id, 'Waiting') if ((job.start_time - 1) <= current_time_sec) and (exit_code == 0): exit_code = 1 # Process toLaunch jobs if 'toLaunch' in jobs_by_state: for job in jobs_by_state['toLaunch']: notify_to_run_job(job.id) logger.debug("End of Meta Scheduler") return exit_code
def check_reservation_jobs(plt, resource_set, queue_name, all_slot_sets, current_time_sec): """Processing of new Advance Reservations""" logger.debug("Queue " + queue_name + ": begin processing of new reservations") ar_jobs_scheduled = {} ar_jobs, ar_jids, nb_ar_jobs = plt.get_waiting_jobs( queue_name, 'toSchedule') logger.debug("nb_ar_jobs:" + str(nb_ar_jobs)) if nb_ar_jobs > 0: job_security_time = int(config['SCHEDULER_JOB_SECURITY_TIME']) plt.get_data_jobs(ar_jobs, ar_jids, resource_set, job_security_time) logger.debug("Try and schedule new reservations") for jid in ar_jids: job = ar_jobs[jid] logger.debug( "Find resource for Advance Reservation job:" + str(job.id)) # It is a reservation, we take care only of the first moldable job moldable_id, walltime, hy_res_rqts = job.mld_res_rqts[0] # test if reservation is too old if current_time_sec >= (job.start_time + walltime): logger.warn( "[" + str(job.id) + "] Canceling job: reservation is too old") set_job_message(job.id, "Reservation too old") set_job_state(job.id, 'toError') continue else: if job.start_time < current_time_sec: # TODO update to DB ???? job.start_time = current_time_sec ss_name = 'default' # TODO container # if 'inner' in job.types: # ss_name = job.types['inner'] # TODO: test if container is an AR job slots = all_slot_sets[ss_name].slots t_e = job.start_time + walltime - job_security_time sid_left, sid_right = get_encompassing_slots( slots, job.start_time, t_e) if job.ts or (job.ph == ALLOW): itvs_avail = intersec_ts_ph_itvs_slots( slots, sid_left, sid_right, job) else: itvs_avail = intersec_itvs_slots(slots, sid_left, sid_right) itvs = find_resource_hierarchies_job( itvs_avail, hy_res_rqts, resource_set.hierarchy) if ('QUOTAS' in config) and (config['QUOTAS'] == 'yes'): nb_res = itvs_size(intersec(itvs, resource_set.default_resource_itvs)) res = check_slots_quotas(slots, sid_left, sid_right, job, nb_res, walltime) (quotas_ok, quotas_msg, rule, value) = res if not quotas_ok: itvs = [] logger.info("Quotas limitaion reached, job:" + str(job.id) + ", " + quotas_msg + ", rule: " + str(rule) + ", value: " + str(value)) set_job_state(job.id, 'toError') set_job_message(job.id, "This advance reservation cannot run due to quotas") if itvs == []: # not enough resource available logger.warn("[" + str(job.id) + "] advance reservation cannot be validated, not enough resources") set_job_state(job.id, 'toError') set_job_message(job.id, "This advance reservation cannot run") else: # The reservation can be scheduled logger.debug( "[" + str(job.id) + "] advance reservation is validated") job.moldable_id = moldable_id job.res_set = itvs ar_jobs_scheduled[job.id] = job # if 'container' in job.types # slot = Slot(1, 0, 0, job.res_set[:], job.start_time, # job.start_time + job.walltime - job_security_time) # slot.show() # slots_sets[job.id] = SlotSet(slot) set_job_state(job.id, 'toAckReservation') set_job_resa_state(job.id, 'Scheduled') if ar_jobs_scheduled != []: logger.debug("Save AR jobs' assignements in database") save_assigns(ar_jobs_scheduled, resource_set) logger.debug("Queue " + queue_name + ": end processing of new reservations")
def handle_waiting_reservation_jobs(queue_name, resource_set, job_security_time, current_time_sec): logger.debug("Queue " + queue_name + ": begin processing accepted Advance Reservations") ar_jobs = get_waiting_scheduled_AR_jobs(queue_name, resource_set, job_security_time, current_time_sec) for job in ar_jobs: moldable_id = job.moldable_id walltime = job.walltime # Test if AR job is expired and handle it if (current_time_sec > (job.start_time + walltime)): logger.warn("[" + str(job.id) + "] set job state to Error: avdance reservation expired and couldn't be started") set_job_state(job.id, 'Error') set_job_message(job.id, "Reservation expired and couldn't be started.") else: # Determine current available ressources avail_res = intersec(resource_set.roid_itvs, job.res_set) # Test if the AR job is waiting to be launched due to nodes' unavailabilities if (avail_res == []) and (job.start_time < current_time_sec): logger.warn("[%s] advance reservation is waiting because no resource is present" % str(job.id)) # Delay launching time set_gantt_job_start_time(moldable_id, current_time_sec + 1) elif (job.start_time < current_time_sec): if (job.start_time + reservation_waiting_timeout) > current_time_sec: if not equal_itvs(avail_res, job.res_set): # The expected ressources are not all available, # wait the specified timeout logger.warn("[" + str(job.id) + "] advance reservation is waiting because not all \ resources are available yet") set_gantt_job_start_time(moldable_id, current_time_sec + 1) else: # It's time to launch the AR job, remove missing ressources missing_resources_itvs = sub_intervals(job.res_set, avail_res) remove_gantt_resource_job(moldable_id, missing_resources_itvs, resource_set) logger.warn("[" + str(job.id) + "remove some resources assigned to this advance reservation, \ because there are not Alive") add_new_event('SCHEDULER_REDUCE_NB_RESSOURCES_FOR_ADVANCE_RESERVATION', job.id, "[MetaSched] Reduce the number of resources for the job " + str(job.id)) nb_res = itvs_size(job.res_set) - itvs_size(missing_resources_itvs) new_message = re.sub(r'R=\d+', 'R=' + str(nb_res), job.message) if new_message != job.message: set_job_message(job.id, new_message) logger.debug("Queue " + queue_name + ": end processing of reservations with missing resources")