Пример #1
0
def test_quotas_one_job_rule_nb_res_1():
    config['QUOTAS'] = 'yes'
    # quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
    # global quotas_rules
    qts.quotas_rules = {('*', '*', '*', '/'): [1, -1, -1]}

    res = [(1, 32)]
    rs.default_resource_itvs = deepcopy(res)

    ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 100))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, queue='default', user='******', project='')
    j1.simple_req(('node', 2), 60, res)

    schedule_id_jobs_ct(all_ss, {1: j1}, hy, [1], 20)

    print(j1.start_time)
    assert j1.res_set == []
Пример #2
0
def test_quotas_two_jobs_job_type_proc():
    config['QUOTAS'] = 'yes'
    _, quotas_file_name = mkstemp()
    config['QUOTAS_FILE'] = quotas_file_name

    # quotas_file = open(quotas_file_name, 'w')
    with open(config['QUOTAS_FILE'], 'w', encoding="utf-8") as quotas_fd:
        quotas_fd.write('{"quotas": {"*,*,yop,*": [-1,1,-1]}, "quotas_job_types": ["yop"]}')

    qts.load_quotas_rules()

    print(qts.quotas_rules, qts.quotas_job_types)

    res = [(1, 32)]
    rs.default_resource_itvs = res

    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, queue='default', user='******', project='', types={'yop'})
    j1.simple_req(('node', 1), 50, res)
    j2 = JobPseudo(id=2, queue='default', user='******', project='', types={'yop'})
    j2.simple_req(('node', 1), 50, res)

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 20)

    print(j1.start_time, j2.start_time)

    assert j1.start_time == 0
    assert j2.start_time == 50
Пример #3
0
def test_quotas_four_jobs_rule_1():

    config['QUOTAS'] = 'yes'
    # quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
    # global quotas_rules
    qts.quotas_rules = {('*', '*', '*', '/'): [16, -1, -1],
                        ('*', 'yop', '*', '*'): [-1, 1, -1]}

    res = [(1, 32)]
    rs.default_resource_itvs = deepcopy(res)

    ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 10000))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, start_time=0, walltime=20,
                   queue='default', user='******', project='',
                   res_set=[(9, 24)], types={}, ts=False, ph=0)
    j2 = JobPseudo(id=2, start_time=0, walltime=50,
                   queue='default', user='******', project='yop',
                   res_set=[(1, 8)])

    j3 = JobPseudo(id=3, queue='default', user='******', project='')
    j3.simple_req(('node', 1), 10, res)

    j4 = JobPseudo(id=4, queue='default', user='******', project='yop')
    j4.simple_req(('node', 1), 60, res)

    set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 5)

    ss.show_slots()
    # pdb.set_trace()
    schedule_id_jobs_ct(all_ss, {3: j3, 4: j4}, hy, [3, 4], 5)

    print(j3.start_time, j4.start_time)

    assert j3.start_time == 20
    assert j3.res_set == [(9, 16)]
    assert j4.start_time == 50
    assert j4.res_set == [(1, 8)]
Пример #4
0
def gantt_init_with_running_jobs(plt, initial_time_sec, job_security_time):

    #
    # Determine Global Resource Intervals and Initial Slot
    #
    resource_set = plt.resource_set()
    initial_slot_set = SlotSet((resource_set.roid_itvs, initial_time_sec))

    logger.debug("Processing of processing of already handled reservations")
    accepted_ar_jids, accepted_ar_jobs = \
        get_waiting_reservations_already_scheduled(resource_set, job_security_time)
    gantt_flush_tables(accepted_ar_jids)

    # TODO Can we remove this step, below ???
    #  why don't use: assigned_resources and job start_time ??? in get_scheduled_jobs ???
    logger.debug("Processing of current jobs")
    current_jobs = get_jobs_in_multiple_states(['Running', 'toLaunch', 'Launching',
                                                'Finishing', 'Suspended', 'Resuming'],
                                               resource_set)
    plt.save_assigns(current_jobs, resource_set)

    #
    #  Resource availabilty (Available_upto field) is integrated through pseudo job
    #
    pseudo_jobs = []
    for t_avail_upto in sorted(resource_set.available_upto.keys()):
        itvs = resource_set.available_upto[t_avail_upto]
        j = JobPseudo()
        j.start_time = t_avail_upto
        j.walltime = MAX_TIME - t_avail_upto
        j.res_set = itvs
        j.ts = False
        j.ph = NO_PLACEHOLDER

        pseudo_jobs.append(j)

    if pseudo_jobs != []:
        initial_slot_set.split_slots_jobs(pseudo_jobs)

    #
    # Get already scheduled jobs advanced reservations and jobs from more higher priority queues
    #
    # TODO?: Remove resources of the type specified in
    # SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
    scheduled_jobs = plt.get_scheduled_jobs(
        resource_set, job_security_time, initial_time_sec)

    # retrieve ressources used by besteffort jobs
    besteffort_rid2job = {}

    for job in scheduled_jobs:
        #  print("job.id:", job.id, job.queue_name, job.types, job.res_set, job.start_time)
        if 'besteffort' in job.types:
            for r_id in itvs2ids(job.res_set):
                besteffort_rid2job[r_id] = job

    # Create and fill gantt
    all_slot_sets = {'default': initial_slot_set}
    if scheduled_jobs != []:
        filter_besteffort = True
        set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs,
                                           job_security_time, initial_time_sec,
                                           filter_besteffort)

    return (all_slot_sets, scheduled_jobs, besteffort_rid2job)
Пример #5
0
def schedule_cycle(plt, queue='default'):
    now = plt.get_time()

    logger.info('Begin scheduling....', now)

    #
    # Retrieve waiting jobs
    #
    waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)

    logger.info(waiting_jobs, waiting_jids, nb_waiting_jobs)

    if nb_waiting_jobs > 0:

        #
        # Determine Global Resource Intervals and Initial Slot
        #
        resource_set = plt.resource_set()
        initial_slot_set = SlotSet((resource_set.roid_itvs, now))

        #
        #  Resource availabilty (Available_upto field) is integrated through pseudo job
        #
        pseudo_jobs = []
        for t_avail_upto in sorted(resource_set.available_upto.keys()):
            itvs = resource_set.available_upto[t_avail_upto]
            j = JobPseudo()
            # logger.info(t_avail_upto, MAX_TIME - t_avail_upto, itvs)
            j.start_time = t_avail_upto
            j.walltime = MAX_TIME - t_avail_upto
            j.res_set = itvs
            j.ts = False
            j.ph = NO_PLACEHOLDER

            pseudo_jobs.append(j)

        if pseudo_jobs != []:
            initial_slot_set.split_slots_jobs(pseudo_jobs)

        #
        # Get  additional waiting jobs' data
        #
        job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])
        plt.get_data_jobs(waiting_jobs, waiting_jids, resource_set, job_security_time)

        #
        # Get already scheduled jobs advanced reservations and jobs from more higher priority queues
        #
        scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)

        if scheduled_jobs != []:
            initial_slot_set.split_slots_jobs(scheduled_jobs)

        # initial_slot_set.show_slots()

        all_slot_sets = {"default": initial_slot_set}

        #
        # Scheduled
        #
        schedule_id_jobs_ct(all_slot_sets,
                            waiting_jobs,
                            resource_set.hierarchy,
                            waiting_jids,
                            0)

        #
        # Save assignement
        #
        plt.save_assigns(waiting_jobs, resource_set)
    else:
        logger.info("no waiting jobs")
Пример #6
0
def schedule_cycle(plt, now, queue="default"):

    logger.info("Begin scheduling....now: " + str(now) + ", queue: " + queue)

    #
    # Retrieve waiting jobs
    #
    waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)

    if nb_waiting_jobs > 0:
        logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs))
        for jid in waiting_jids:
            logger.debug("waiting_jid: " + str(jid))

        job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])

        #
        # Determine Global Resource Intervals and Initial Slot
        #
        resource_set = plt.resource_set()
        initial_slot_set = SlotSet((resource_set.roid_itvs, now))

        #
        #  Resource availabilty (Available_upto field) is integrated through pseudo job
        #
        pseudo_jobs = []
        for t_avail_upto in sorted(resource_set.available_upto.keys()):
            itvs = resource_set.available_upto[t_avail_upto]
            j = JobPseudo()
            # print t_avail_upto, max_time - t_avail_upto, itvs
            j.start_time = t_avail_upto
            j.walltime = MAX_TIME - t_avail_upto
            j.res_set = itvs
            j.ts = False
            j.ph = NO_PLACEHOLDER

            pseudo_jobs.append(j)

        if pseudo_jobs != []:
            initial_slot_set.split_slots_jobs(pseudo_jobs)
            
        #
        # Get  additional waiting jobs' data
        #
        plt.get_data_jobs(
            waiting_jobs, waiting_jids, resource_set, job_security_time)

        # Job sorting (karma and advanced)
        waiting_ordered_jids = karma_job_sorting(queue, now, waiting_jids, waiting_jobs, plt)

        #
        # Get already scheduled jobs advanced reservations and jobs from more higher priority queues
        #
        scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)

        all_slot_sets = {'default': initial_slot_set}

        if scheduled_jobs != []:
            if queue == 'besteffort':
                filter_besteffort = False
            else:
                filter_besteffort = True
            set_slots_with_prev_scheduled_jobs(all_slot_sets,
                                               scheduled_jobs,
                                               job_security_time,
                                               now,
                                               filter_besteffort)
        #
        # Scheduled
        #
        schedule_id_jobs_ct(all_slot_sets,
                            waiting_jobs,
                            resource_set.hierarchy,
                            waiting_ordered_jids,
                            job_security_time)

        #
        # Save assignement
        #
        logger.info("save assignement")

        plt.save_assigns(waiting_jobs, resource_set)
    else:
        logger.info("no waiting jobs")