Example #1
0
def test_bug_split_slots():

    v = [(20, 69, [(31, 32)]),
         (70, 79, [(1, 15), (31, 32)]),
         (80, 2147483599, [(1, 32)]),
         (2147483600, 2147483647, [])
         ]

    # res = [(1, 32)]
    s1 = Slot(1, 0, 4, [(16, 32)], 20, 69)
    s2 = Slot(2, 1, 0, [], 2147483600, 2147483647)
    s4 = Slot(4, 1, 2, [(1, 32)], 70, 2147483599)

    slts = dict(((s.id, s) for s in [s1, s2, s4]))
    ss = SlotSet(slts)

    j2 = JobPseudo(id=2,
                   start_time=20,
                   walltime=60,
                   res_set=[(16, 30)],
                   ts=False,
                   ph=0)

    ss.split_slots(1, 4, j2)
    assert compare_slots_val_ref(ss.slots, v)
Example #2
0
def test_add_split_slots_jobs_2_jobs_2():
    v = [(10, 19, []),
         (20, 99, [(40, 50)]),
         (100, 129, [(10, 20), (40, 50)]),
         (130, 219, [(40, 50)]),
         (220, MAX_TIME, []),
         ]

    ss = SlotSet(([], 10))

    j1 = JobPseudo(id=1,
                   start_time=100,
                   walltime=30,
                   res_set=[(10, 20)],
                   ts=False,
                   ph=0)

    j2 = JobPseudo(id=2,
                   start_time=20,
                   walltime=200,
                   res_set=[(40, 50)],
                   ts=False,
                   ph=0)

    ss.split_slots_jobs([j2, j1], False)

    assert compare_slots_val_ref(ss.slots, v)
Example #3
0
def test_assign_resources_mld_job_split_slots_2():

    v = [(0, 59, [(17, 21)]), (60, 100, [(1, 32)])]

    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]], "switch": [[(1, 16)], [(17, 21)]], "gpu": [[(22, 32)]]}

    j1 = JobPseudo(id=1, key_cache={}, mld_res_rqts=[(1, 60, [([("node", 2)], res), ([("gpu", 1)], res)])])

    assign_resources_mld_job_split_slots(ss, j1, hy, -1)
    ss.show_slots()
    assert compare_slots_val_ref(ss.slots, v)
Example #4
0
def test_split_slots_bc():
    v = [(1, 10, [(1, 9), (21, 32)]), (11, 20, [(1, 32)])]

    j1 = JobPseudo(id=1,
                   start_time=1,
                   walltime=10,
                   res_set=[(10, 20)],
                   moldable_id=1,
                   ts=False,
                   ph=0)

    ss = SlotSet(Slot(1, 0, 0, [(1, 32)], 1, 20))
    ss.split_slots(1, 1, j1)
    assert compare_slots_val_ref(ss.slots, v)
Example #5
0
def test_add_split_slots_jobs_one_job():

    v = [(10, 14, [(10, 50)]), (15, MAX_TIME, [])]

    ss = SlotSet(([], 10))

    j = JobPseudo(id=1,
                  start_time=5,
                  walltime=10,
                  res_set=[(10, 50)],
                  ts=False,
                  ph=0)

    ss.split_slots_jobs([j], False)

    assert compare_slots_val_ref(ss.slots, v)
Example #6
0
def test_quotas_four_jobs_rule_1():

    config['QUOTAS'] = 'yes'
    # quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
    # global quotas_rules
    qts.quotas_rules = {('*', '*', '*', '/'): [16, -1, -1],
                        ('*', 'yop', '*', '*'): [-1, 1, -1]}

    res = [(1, 32)]
    rs.default_resource_itvs = deepcopy(res)

    ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 10000))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, start_time=0, walltime=20,
                   queue='default', user='******', project='',
                   res_set=[(9, 24)], types={}, ts=False, ph=0)
    j2 = JobPseudo(id=2, start_time=0, walltime=50,
                   queue='default', user='******', project='yop',
                   res_set=[(1, 8)])

    j3 = JobPseudo(id=3, queue='default', user='******', project='')
    j3.simple_req(('node', 1), 10, res)

    j4 = JobPseudo(id=4, queue='default', user='******', project='yop')
    j4.simple_req(('node', 1), 60, res)

    set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 5)

    ss.show_slots()
    # pdb.set_trace()
    schedule_id_jobs_ct(all_ss, {3: j3, 4: j4}, hy, [3, 4], 5)

    print(j3.start_time, j4.start_time)

    assert j3.start_time == 20
    assert j3.res_set == [(9, 16)]
    assert j4.start_time == 50
    assert j4.res_set == [(1, 8)]
Example #7
0
def gantt_init_with_running_jobs(plt, initial_time_sec, job_security_time):

    #
    # Determine Global Resource Intervals and Initial Slot
    #
    resource_set = plt.resource_set()
    initial_slot_set = SlotSet((resource_set.roid_itvs, initial_time_sec))

    logger.debug("Processing of processing of already handled reservations")
    accepted_ar_jids, accepted_ar_jobs = \
        get_waiting_reservations_already_scheduled(resource_set, job_security_time)
    gantt_flush_tables(accepted_ar_jids)

    # TODO Can we remove this step, below ???
    #  why don't use: assigned_resources and job start_time ??? in get_scheduled_jobs ???
    logger.debug("Processing of current jobs")
    current_jobs = get_jobs_in_multiple_states(['Running', 'toLaunch', 'Launching',
                                                'Finishing', 'Suspended', 'Resuming'],
                                               resource_set)
    plt.save_assigns(current_jobs, resource_set)

    #
    #  Resource availabilty (Available_upto field) is integrated through pseudo job
    #
    pseudo_jobs = []
    for t_avail_upto in sorted(resource_set.available_upto.keys()):
        itvs = resource_set.available_upto[t_avail_upto]
        j = JobPseudo()
        j.start_time = t_avail_upto
        j.walltime = MAX_TIME - t_avail_upto
        j.res_set = itvs
        j.ts = False
        j.ph = NO_PLACEHOLDER

        pseudo_jobs.append(j)

    if pseudo_jobs != []:
        initial_slot_set.split_slots_jobs(pseudo_jobs)

    #
    # Get already scheduled jobs advanced reservations and jobs from more higher priority queues
    #
    # TODO?: Remove resources of the type specified in
    # SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
    scheduled_jobs = plt.get_scheduled_jobs(
        resource_set, job_security_time, initial_time_sec)

    # retrieve ressources used by besteffort jobs
    besteffort_rid2job = {}

    for job in scheduled_jobs:
        #  print("job.id:", job.id, job.queue_name, job.types, job.res_set, job.start_time)
        if 'besteffort' in job.types:
            for r_id in itvs2ids(job.res_set):
                besteffort_rid2job[r_id] = job

    # Create and fill gantt
    all_slot_sets = {'default': initial_slot_set}
    if scheduled_jobs != []:
        filter_besteffort = True
        set_slots_with_prev_scheduled_jobs(all_slot_sets, scheduled_jobs,
                                           job_security_time, initial_time_sec,
                                           filter_besteffort)

    return (all_slot_sets, scheduled_jobs, besteffort_rid2job)
Example #8
0
def schedule_cycle(plt, queue='default'):
    now = plt.get_time()

    logger.info('Begin scheduling....', now)

    #
    # Retrieve waiting jobs
    #
    waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)

    logger.info(waiting_jobs, waiting_jids, nb_waiting_jobs)

    if nb_waiting_jobs > 0:

        #
        # Determine Global Resource Intervals and Initial Slot
        #
        resource_set = plt.resource_set()
        initial_slot_set = SlotSet((resource_set.roid_itvs, now))

        #
        #  Resource availabilty (Available_upto field) is integrated through pseudo job
        #
        pseudo_jobs = []
        for t_avail_upto in sorted(resource_set.available_upto.keys()):
            itvs = resource_set.available_upto[t_avail_upto]
            j = JobPseudo()
            # logger.info(t_avail_upto, MAX_TIME - t_avail_upto, itvs)
            j.start_time = t_avail_upto
            j.walltime = MAX_TIME - t_avail_upto
            j.res_set = itvs
            j.ts = False
            j.ph = NO_PLACEHOLDER

            pseudo_jobs.append(j)

        if pseudo_jobs != []:
            initial_slot_set.split_slots_jobs(pseudo_jobs)

        #
        # Get  additional waiting jobs' data
        #
        job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])
        plt.get_data_jobs(waiting_jobs, waiting_jids, resource_set, job_security_time)

        #
        # Get already scheduled jobs advanced reservations and jobs from more higher priority queues
        #
        scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)

        if scheduled_jobs != []:
            initial_slot_set.split_slots_jobs(scheduled_jobs)

        # initial_slot_set.show_slots()

        all_slot_sets = {"default": initial_slot_set}

        #
        # Scheduled
        #
        schedule_id_jobs_ct(all_slot_sets,
                            waiting_jobs,
                            resource_set.hierarchy,
                            waiting_jids,
                            0)

        #
        # Save assignement
        #
        plt.save_assigns(waiting_jobs, resource_set)
    else:
        logger.info("no waiting jobs")
Example #9
0
def schedule_cycle(plt, now, queue="default"):

    logger.info("Begin scheduling....now: " + str(now) + ", queue: " + queue)

    #
    # Retrieve waiting jobs
    #
    waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)

    if nb_waiting_jobs > 0:
        logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs))
        for jid in waiting_jids:
            logger.debug("waiting_jid: " + str(jid))

        job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])

        #
        # Determine Global Resource Intervals and Initial Slot
        #
        resource_set = plt.resource_set()
        initial_slot_set = SlotSet((resource_set.roid_itvs, now))

        #
        #  Resource availabilty (Available_upto field) is integrated through pseudo job
        #
        pseudo_jobs = []
        for t_avail_upto in sorted(resource_set.available_upto.keys()):
            itvs = resource_set.available_upto[t_avail_upto]
            j = JobPseudo()
            # print t_avail_upto, max_time - t_avail_upto, itvs
            j.start_time = t_avail_upto
            j.walltime = MAX_TIME - t_avail_upto
            j.res_set = itvs
            j.ts = False
            j.ph = NO_PLACEHOLDER

            pseudo_jobs.append(j)

        if pseudo_jobs != []:
            initial_slot_set.split_slots_jobs(pseudo_jobs)
            
        #
        # Get  additional waiting jobs' data
        #
        plt.get_data_jobs(
            waiting_jobs, waiting_jids, resource_set, job_security_time)

        # Job sorting (karma and advanced)
        waiting_ordered_jids = karma_job_sorting(queue, now, waiting_jids, waiting_jobs, plt)

        #
        # Get already scheduled jobs advanced reservations and jobs from more higher priority queues
        #
        scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)

        all_slot_sets = {'default': initial_slot_set}

        if scheduled_jobs != []:
            if queue == 'besteffort':
                filter_besteffort = False
            else:
                filter_besteffort = True
            set_slots_with_prev_scheduled_jobs(all_slot_sets,
                                               scheduled_jobs,
                                               job_security_time,
                                               now,
                                               filter_besteffort)
        #
        # Scheduled
        #
        schedule_id_jobs_ct(all_slot_sets,
                            waiting_jobs,
                            resource_set.hierarchy,
                            waiting_ordered_jids,
                            job_security_time)

        #
        # Save assignement
        #
        logger.info("save assignement")

        plt.save_assigns(waiting_jobs, resource_set)
    else:
        logger.info("no waiting jobs")