Example #1
0
def test_quotas_one_job_rule_nb_res_2():

    config['QUOTAS'] = 'yes'
    # quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
    # global quotas_rules
    qts.quotas_rules = {('*', '*', '*', '/'): [16, -1, -1]}

    res = [(1, 32)]
    rs.default_resource_itvs = res

    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=2, types={}, deps=[], key_cache={},
                   queue='default', user='******', project='',
                   mld_res_rqts=[
        (1, 60,
         [([("node", 2)], res)]
         )
    ], ts=False, ph=0)

    schedule_id_jobs_ct(all_ss, {1: j1}, hy, [1], 20)

    assert j1.res_set == [(1, 16)]
Example #2
0
def test_schedule_container_error3():
    """ inner exceeds time container's capacity"""

    res = [(1, 32)]

    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1,
        types={"container": ""},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 60, [([("node", 2)], res[:])])],
        ts=False,
        ph=0,
    )

    j2 = JobPseudo(
        id=2,
        types={"inner": "1"},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 70, [([("node", 1)], res[:])])],
        ts=False,
        ph=0,
    )

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 20)

    assert j2.start_time == -1
Example #3
0
def test_quotas_two_jobs_job_type_proc():
    config['QUOTAS'] = 'yes'
    _, quotas_file_name = mkstemp()
    config['QUOTAS_FILE'] = quotas_file_name

    # quotas_file = open(quotas_file_name, 'w')
    with open(config['QUOTAS_FILE'], 'w', encoding="utf-8") as quotas_fd:
        quotas_fd.write('{"quotas": {"*,*,yop,*": [-1,1,-1]}, "quotas_job_types": ["yop"]}')

    qts.load_quotas_rules()

    print(qts.quotas_rules, qts.quotas_job_types)

    res = [(1, 32)]
    rs.default_resource_itvs = res

    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, queue='default', user='******', project='', types={'yop'})
    j1.simple_req(('node', 1), 50, res)
    j2 = JobPseudo(id=2, queue='default', user='******', project='', types={'yop'})
    j2.simple_req(('node', 1), 50, res)

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 20)

    print(j1.start_time, j2.start_time)

    assert j1.start_time == 0
    assert j2.start_time == 50
Example #4
0
def test_schedule_container1():

    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1,
        types={"container": ""},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 80, [([("node", 2)], res[:])])],
        ts=False,
        ph=0,
    )

    j2 = JobPseudo(
        id=2,
        types={"inner": "1"},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 30, [([("node", 1)], res[:])])],
        ts=False,
        ph=0,
    )

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 10)

    j2.res_set, [(1, 8)]
Example #5
0
def test_schedule_placeholder2():
    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1, types={}, deps=[], key_cache={}, mld_res_rqts=[(1, 60, [([("node", 2)], res[:])])], ts=False, ph=0
    )

    # job type: allow="yop"
    j2 = JobPseudo(
        id=2,
        types={},
        deps=[(1, "Waiting", 0)],
        key_cache={},
        mld_res_rqts=[(1, 80, [([("node", 2)], res[:])])],
        ts=False,
        ph=2,
        ph_name="yop",
    )

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 20)

    print("j1.start_time:", j1.start_time, " j2.start_time:", j2.start_time)

    assert j2.start_time == 60
Example #6
0
def test_schedule_placeholder_prev_sched():

    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1,
        types={},
        deps=[],
        key_cache={},
        res_set=[(7, 27)],
        start_time=200,
        walltime=150,
        mld_res_rqts=[(1, 60, [([("node", 2)], res[:])])],
        ts=False,
        ph=1,
        ph_name="yop",
    )

    j2 = JobPseudo(
        id=2, types={}, deps=[], key_cache={}, mld_res_rqts=[(1, 150, [([("node", 4)], res[:])])], ts=False, ph=0
    )

    j3 = JobPseudo(
        id=3,
        types={},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 500, [([("node", 2)], res[:])])],
        ts=False,
        ph=2,
        ph_name="yop",
    )

    # pdb.set_trace()
    set_slots_with_prev_scheduled_jobs(all_ss, [j1], 20)

    all_ss["default"].show_slots()
    # pdb.set_trace()

    schedule_id_jobs_ct(all_ss, {2: j2, 3: j3}, hy, [2, 3], 20)

    print("j1.start_time:", j1.start_time, "j2.start_time:", j2.start_time, " j3.start_time:", j3.start_time)

    all_ss["default"].show_slots()
    # pdb.set_trace()

    # assert j3.start_time == 150
    assert j3.res_set == [(1, 16)]
Example #7
0
def test_schedule_w_temporally_fragmented_container():
    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 5000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1,
        types={"container": "yop"},
        deps=[],
        key_cache={},
        res_set=[(7, 32)],
        start_time=200,
        walltime=50,
        ts=False,
        ph=0,
    )

    j2 = JobPseudo(
        id=2,
        types={"container": "yop"},
        deps=[],
        key_cache={},
        res_set=[(15, 25)],
        start_time=1000,
        walltime=200,
        ts=False,
        ph=0,
    )

    j3 = JobPseudo(
        id=3,
        types={"inner": "yop"},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 100, [([("node", 1)], res[:])])],
        ts=False,
        ph=0,
    )

    set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 20)
    all_ss["yop"].show_slots()

    schedule_id_jobs_ct(all_ss, {3: j3}, hy, [3], 20)

    all_ss["yop"].show_slots()

    assert j3.start_time == 1000
    assert j3.res_set == [(17, 24)]
Example #8
0
def test_schedule_container_prev_sched():

    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1,
        types={"container": ""},
        deps=[],
        key_cache={},
        res_set=[(7, 27)],
        start_time=200,
        walltime=150,
        mld_res_rqts=[(1, 60, [([("node", 2)], res[:])])],
        ts=False,
        ph=0,
    )

    j2 = JobPseudo(
        id=2,
        types={"inner": "1"},
        deps=[],
        key_cache={},
        res_set=[(9, 16)],
        start_time=210,
        walltime=70,
        mld_res_rqts=[(1, 30, [([("node", 1)], res[:])])],
        ts=False,
        ph=0,
    )

    j3 = JobPseudo(
        id=3,
        types={"inner": "1"},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 30, [([("node", 1)], res[:])])],
        ts=False,
        ph=0,
    )

    set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 20)

    schedule_id_jobs_ct(all_ss, {3: j3}, hy, [3], 20)

    assert j3.start_time == 200
    assert j3.res_set == [(17, 24)]
Example #9
0
def test_schedule_id_jobs_ct_1():
    v = [(0, 59, [(17, 32)]), (60, 100, [(1, 32)])]

    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1, types={}, deps=[], key_cache={}, mld_res_rqts=[(1, 60, [([("node", 2)], res)])], ts=False, ph=0
    )

    schedule_id_jobs_ct(all_ss, {1: j1}, hy, [1], 20)

    assert compare_slots_val_ref(ss.slots, v) is True
Example #10
0
def test_terminated_dependency():
    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j2 = JobPseudo(
        id=2,
        types={},
        deps=[(1, "Terminated", 0)],
        key_cache={},
        mld_res_rqts=[(1, 80, [([("node", 2)], res[:])])],
        ts=False,
        ph=0,
    )

    schedule_id_jobs_ct(all_ss, {2: j2}, hy, [2], 20)

    assert j2.start_time == 0
Example #11
0
def test_schedule_error_1():
    # Be careful you need a deepcopy for resources constraint when declare
    res = [(1, 32)]

    ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 10000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j2 = JobPseudo(id=2, start_time=0, walltime=30, res_set=[(1, 8)], types={}, ts=False, ph=0)

    set_slots_with_prev_scheduled_jobs(all_ss, [j2], 10)

    j4 = JobPseudo(
        id=4, types={}, deps=[], key_cache={}, mld_res_rqts=[(1, 60, [([("node", 2)], deepcopy(res))])], ts=False, ph=0
    )

    schedule_id_jobs_ct(all_ss, {4: j4}, hy, [4], 5)

    assert j4.res_set == [(9, 24)]
Example #12
0
def test_quotas_one_job_rule_nb_res_1():
    config['QUOTAS'] = 'yes'
    # quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
    # global quotas_rules
    qts.quotas_rules = {('*', '*', '*', '/'): [1, -1, -1]}

    res = [(1, 32)]
    rs.default_resource_itvs = deepcopy(res)

    ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 100))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, queue='default', user='******', project='')
    j1.simple_req(('node', 2), 60, res)

    schedule_id_jobs_ct(all_ss, {1: j1}, hy, [1], 20)

    print(j1.start_time)
    assert j1.res_set == []
Example #13
0
def test_schedule_placeholder1():
    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    # job type: placeholder="yop"
    j1 = JobPseudo(
        id=1,
        types={},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 80, [([("node", 4)], res[:])])],
        ts=False,
        ph=1,
        ph_name="yop",
    )

    j2 = JobPseudo(
        id=2, types={}, deps=[], key_cache={}, mld_res_rqts=[(1, 50, [([("node", 4)], res[:])])], ts=False, ph=0
    )

    # Allow type: allow="yop"
    j3 = JobPseudo(
        id=3,
        types={},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 60, [([("node", 4)], res[:])])],
        ts=False,
        ph=2,
        ph_name="yop",
    )

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2, 3: j3}, hy, [1, 2, 3], 20)

    print(
        "Placeholder: j1.start_time:", j1.start_time, " j2.start_time:", j2.start_time, " j3.start_time:", j3.start_time
    )

    assert (j2.start_time == 80) and (j3.start_time == 0)
Example #14
0
def test_quotas_one_job_no_rules():
    config['QUOTAS'] = 'yes'

    v = [(0, 59, [(17, 32)]), (60, 100, [(1, 32)])]

    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, types={}, deps=[], key_cache={},
                   queue='default', user='******', project='',
                   mld_res_rqts=[
        (1, 60,
         [([("node", 2)], res)]
         )
    ], ts=False, ph=0)

    schedule_id_jobs_ct(all_ss, {1: j1}, hy, [1], 20)

    assert compare_slots_val_ref(ss.slots, v)
Example #15
0
def test_quotas_four_jobs_rule_1():

    config['QUOTAS'] = 'yes'
    # quotas.set_quotas_rules({('*', '*', '*', '/'): [1, -1, -1]})
    # global quotas_rules
    qts.quotas_rules = {('*', '*', '*', '/'): [16, -1, -1],
                        ('*', 'yop', '*', '*'): [-1, 1, -1]}

    res = [(1, 32)]
    rs.default_resource_itvs = deepcopy(res)

    ss = SlotSet(Slot(1, 0, 0, deepcopy(res), 0, 10000))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, start_time=0, walltime=20,
                   queue='default', user='******', project='',
                   res_set=[(9, 24)], types={}, ts=False, ph=0)
    j2 = JobPseudo(id=2, start_time=0, walltime=50,
                   queue='default', user='******', project='yop',
                   res_set=[(1, 8)])

    j3 = JobPseudo(id=3, queue='default', user='******', project='')
    j3.simple_req(('node', 1), 10, res)

    j4 = JobPseudo(id=4, queue='default', user='******', project='yop')
    j4.simple_req(('node', 1), 60, res)

    set_slots_with_prev_scheduled_jobs(all_ss, [j1, j2], 5)

    ss.show_slots()
    # pdb.set_trace()
    schedule_id_jobs_ct(all_ss, {3: j3, 4: j4}, hy, [3, 4], 5)

    print(j3.start_time, j4.start_time)

    assert j3.start_time == 20
    assert j3.res_set == [(9, 16)]
    assert j4.start_time == 50
    assert j4.res_set == [(1, 8)]
Example #16
0
def test_quotas_two_job_rules_nb_res_quotas_file():

    config['QUOTAS'] = 'yes'
    _, quotas_file_name = mkstemp()
    config['QUOTAS_FILE'] = quotas_file_name

    # quotas_file = open(quotas_file_name, 'w')
    with open(config['QUOTAS_FILE'], 'w', encoding="utf-8") as quotas_fd:
        quotas_fd.write('{"quotas": {"*,*,*,toto": [1,-1,-1],"*,*,*,john": [150,-1,-1]}}')

    qts.load_quotas_rules()

    res = [(1, 32)]
    rs.default_resource_itvs = res

    ss = SlotSet(Slot(1, 0, 0, res, 0, 100))
    all_ss = {"default": ss}
    hy = {'node': [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(id=1, types={}, deps=[], key_cache={},
                   queue='default', user='******', project='',
                   mld_res_rqts=[
        (1, 60,
         [([("node", 2)], res)]
         )
    ], ts=False, ph=0)

    j2 = JobPseudo(id=2, types={}, deps=[], key_cache={},
                   queue='default', user='******', project='',
                   mld_res_rqts=[
        (1, 60,
         [([("node", 2)], res)]
         )
    ], ts=False, ph=0)

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 20)

    assert j1.res_set == []
    assert j2.res_set == [(1, 16)]
Example #17
0
def internal_schedule_cycle(plt, now, all_slot_sets, job_security_time, queue):

    resource_set = plt.resource_set()

    #
    # Retrieve waiting jobs
    #
    waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)

    if nb_waiting_jobs > 0:
        logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs))
        for jid in waiting_jids:
            logger.debug("waiting_jid: " + str(jid))

        #
        # Get  additional waiting jobs' data
        #
        plt.get_data_jobs(
            waiting_jobs, waiting_jids, resource_set, job_security_time)

        waiting_ordered_jids = karma_job_sorting(queue, now, waiting_jids, waiting_jobs, plt)

        #
        # Scheduled
        #
        schedule_id_jobs_ct(all_slot_sets,
                            waiting_jobs,
                            resource_set.hierarchy,
                            waiting_ordered_jids,
                            job_security_time)

        #
        # Save assignement
        #
        logger.info("save assignement")

        plt.save_assigns(waiting_jobs, resource_set)
    else:
        logger.info("no waiting jobs")
Example #18
0
def test_schedule_timesharing1():
    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1,
        types={},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 60, [([("node", 4)], res[:])])],
        user="******",
        name="yop",
        ts=True,
        ts_user="******",
        ts_name="*",
        ph=0,
    )

    j2 = JobPseudo(
        id=2,
        types={},
        deps=[],
        key_cache={},
        mld_res_rqts=[(1, 80, [([("node", 4)], res[:])])],
        user="******",
        name="yop",
        ts=True,
        ts_user="******",
        ts_name="*",
        ph=0,
    )

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 20)

    print("j1.start_time:", j1.start_time, " j2.start_time:", j2.start_time)

    assert j1.start_time == j2.start_time
Example #19
0
def test_error_dependency():
    res = [(1, 32)]
    ss = SlotSet(Slot(1, 0, 0, res, 0, 1000))
    all_ss = {"default": ss}
    hy = {"node": [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]}

    j1 = JobPseudo(
        id=1, types={}, deps=[], key_cache={}, mld_res_rqts=[(1, 60, [([("node", 2)], res[:])])], ts=False, ph=0
    )

    j2 = JobPseudo(
        id=2,
        types={},
        deps=[(1, "Error", 0)],
        key_cache={},
        mld_res_rqts=[(1, 80, [([("node", 2)], res[:])])],
        ts=False,
        ph=0,
    )

    schedule_id_jobs_ct(all_ss, {1: j1, 2: j2}, hy, [1, 2], 20)

    assert not hasattr(j2, "start_time")
Example #20
0
def eva_sched_foo(all_ss, jobs, hy, j_ids):
    schedule_id_jobs_ct(all_ss, jobs, hy, j_ids, 10)
Example #21
0
def schedule_cycle(plt, now, queue="default"):

    logger.info("Begin scheduling....now: " + str(now) + ", queue: " + queue)

    #
    # Retrieve waiting jobs
    #
    waiting_jobs, waiting_jids, nb_waiting_jobs = plt.get_waiting_jobs(queue)

    if nb_waiting_jobs > 0:
        logger.info("nb_waiting_jobs:" + str(nb_waiting_jobs))
        for jid in waiting_jids:
            logger.debug("waiting_jid: " + str(jid))

        job_security_time = int(config["SCHEDULER_JOB_SECURITY_TIME"])

        #
        # Determine Global Resource Intervals and Initial Slot
        #
        resource_set = plt.resource_set()
        initial_slot_set = SlotSet((resource_set.roid_itvs, now))

        #
        #  Resource availabilty (Available_upto field) is integrated through pseudo job
        #
        pseudo_jobs = []
        for t_avail_upto in sorted(resource_set.available_upto.keys()):
            itvs = resource_set.available_upto[t_avail_upto]
            j = JobPseudo()
            # print t_avail_upto, max_time - t_avail_upto, itvs
            j.start_time = t_avail_upto
            j.walltime = MAX_TIME - t_avail_upto
            j.res_set = itvs
            j.ts = False
            j.ph = NO_PLACEHOLDER

            pseudo_jobs.append(j)

        if pseudo_jobs != []:
            initial_slot_set.split_slots_jobs(pseudo_jobs)
            
        #
        # Get  additional waiting jobs' data
        #
        plt.get_data_jobs(
            waiting_jobs, waiting_jids, resource_set, job_security_time)

        # Job sorting (karma and advanced)
        waiting_ordered_jids = karma_job_sorting(queue, now, waiting_jids, waiting_jobs, plt)

        #
        # Get already scheduled jobs advanced reservations and jobs from more higher priority queues
        #
        scheduled_jobs = plt.get_scheduled_jobs(resource_set, job_security_time, now)

        all_slot_sets = {'default': initial_slot_set}

        if scheduled_jobs != []:
            if queue == 'besteffort':
                filter_besteffort = False
            else:
                filter_besteffort = True
            set_slots_with_prev_scheduled_jobs(all_slot_sets,
                                               scheduled_jobs,
                                               job_security_time,
                                               now,
                                               filter_besteffort)
        #
        # Scheduled
        #
        schedule_id_jobs_ct(all_slot_sets,
                            waiting_jobs,
                            resource_set.hierarchy,
                            waiting_ordered_jids,
                            job_security_time)

        #
        # Save assignement
        #
        logger.info("save assignement")

        plt.save_assigns(waiting_jobs, resource_set)
    else:
        logger.info("no waiting jobs")