def test_cstring() -> None:
c = Complex("dept", "d", "CSTRING", "<=", True, False, "ABC", 0)
assert c.name == "dept"
assert c.shortcut == "d"
assert c.default == "abc"
assert c.parse("MixedCase") == "mixedcase"
assert c.parse(None) is None
def test_other_strings() -> None:
for stype in ["RESTRING", "TIME", "STRING", "HOST"]:
c = Complex("compy", "c", stype, "<=", True, False, "ABC", 0)
assert c.name == "compy"
assert c.shortcut == "c"
assert c.default == "ABC"
assert c.parse("MixedCase") == "MixedCase"
assert c.parse(None) is None
def test_double_parsing() -> None:
c = Complex("disk", "d", "DOUBLE", "<=", True, True, "123.123", 0)
assert c.name == "disk"
assert c.shortcut == "d"
assert c.default == 123.123
assert c.parse("234.234") == 234.234
assert c.parse("infinity") == float("inf")
for expr in ["SDF", None, ""]:
assert c.parse(expr) == expr
def test_int_parsing() -> None:
for itype in ["INT", "RSMAP"]:
c = Complex("slots", "s", itype, "<=", True, True, "1", 1000)
assert c.parse("100") == 100
assert c.default == 1
assert c.name == "slots"
assert c.shortcut == "s"
for expr in ["100.1", "a", None, ""]:
c.parse(expr) == expr
def run_test(
ctype: str,
node_lic: Optional[bool],
hg_lic: bool,
q_default_lic: bool,
complex_default: Optional[bool],
) -> SchedulerNode:
node_res = {}
if node_lic is not None:
node_res["lic"] = node_lic
node_res["l"] = node_lic
node = SchedulerNode("tux", node_res)
ge_env = common_ge_env()
q = ge_env.queues["hpc.q"]
complex_default_str = (
"NONE" if complex_default is None else str(complex_default)
)
ge_env.complexes["lic"] = Complex(
"lic", "l", ctype, "<=", True, True, complex_default_str, 0
)
q.complex_values[None] = {"lic": q_default_lic}
q.complex_values["@hpc.q"] = {"lic": hg_lic}
assert node.available.get("lic") == node_lic
process_quotas(node, ge_env.complexes, ["@hpc.q"], [q])
return node
def run_test(
ctype: str,
node_pcpu: Optional[N],
hg_pcpu: N,
q_default_pcpu: N,
complex_default: Optional[N],
) -> SchedulerNode:
cast = float if ctype == "DOUBLE" else int
node_res = {}
if node_pcpu is not None:
node_res["pcpu"] = cast(node_pcpu)
node_res["p"] = cast(node_pcpu)
node = SchedulerNode("tux", node_res)
ge_env = common_ge_env()
q = ge_env.queues["hpc.q"]
complex_default_str = (
"NONE" if complex_default is None else str(complex_default)
)
ge_env.complexes["pcpu"] = Complex(
"pcpu", "p", ctype, "<=", True, True, complex_default_str, 0
)
q.complex_values[None] = {"pcpu": cast(q_default_pcpu)}
q.complex_values["@hpc.q"] = {"pcpu": cast(hg_pcpu)}
assert node.available.get("pcpu") == node_pcpu
process_quotas(node, ge_env.complexes, ["@hpc.q"], [q])
return node
def test_complex_parsing_of_queue() -> None:
pcpu = Complex("pcpu", "pcpu", "INT", "<=", True, True, "1", 0)
pmem = Complex("pmem", "pmem", "INT", "<=", True, True, "1", 0)
ldf = Complex("ldf", "ldf", "BOOL", "<=", True, True, "1", 0)
c = {"pcpu": pcpu, "pmem": pmem, "ldf": ldf}
f = parse_queue_complex_values
assert {
None: {},
"@ldek5": {"pcpu": 24, "pmem": 376, "ldf": True},
"@lhaa5": {"pcpu": 4, "pmem": 32},
} == f("None,[@ldek5=pcpu=24,pmem=376,ldf=1],[@lhaa5=pcpu=4,pmem=32]", c, "q1")
assert {
None: {"pcpu": 2, "pmem": 4},
"@ldek5": {"pcpu": 24, "pmem": 376, "ldf": True},
"@lhaa5": {"pcpu": 4, "pmem": 32},
} == f(
"pcpu=2,pmem=4,[@ldek5=pcpu=24,pmem=376,ldf=1],[@lhaa5=pcpu=4,pmem=32]", c, "q1"
)
def test_hostlist() -> None:
"seqno, complex_values, slots, users..."
pes = {
"make":
ParallelEnvironment({
"pe_name": "make",
"allocation_rule": "1"
}),
"mpi":
ParallelEnvironment({
"pe_name": "mpi",
"allocation_rule": "$round_robin"
}),
}
queue_config = {
"qname": "testq",
"hostlist": "@hostlisthg",
"user_lists": "[@userlisthg=user]",
"xuser_lists": "[@xuserlisthg=user]",
"projects": "[@projecthg=prj]",
"xprojects": "[@xprojecthg=prj]",
"seq_no": "0,[@seqnohg=100]",
"complex_values": "pcpu=2,[@complexvalueshg=pcpu=1]",
"pe_list": "make,[@pelisthg=mpi]",
}
unbound_hostgroups = {
"@hostlisthg": Hostgroup("@hostlisthg"),
"@userlisthg": Hostgroup("@userlisthg"),
"@xuserlisthg": Hostgroup("@xuserlisthg"),
"@projecthg": Hostgroup("@projecthg"),
"@xprojecthg": Hostgroup("@xprojecthg"),
"@seqnohg": Hostgroup("@seqnohg"),
"@complexvalueshg": Hostgroup("@complexvalueshg"),
"@pelisthg": Hostgroup("@pelisthg"),
"@notreferenced": Hostgroup("@notreferenced"),
}
complex_values = {None: {"pcpu": 2}, "@complexvalueshg": {"pcpu": 1}}
scheduler = GridEngineScheduler({}, "localhost")
queue = GridEngineQueue(queue_config, scheduler, pes, unbound_hostgroups,
complex_values)
assert "@complexvalueshg" in queue.complex_values
unreferenced = set(unbound_hostgroups.keys()) - set(queue.hostlist)
assert unreferenced == set(["@notreferenced"])
pcpu_c = Complex("pcpu", "p", "INT", "<=", True, True, "123", 100)
assert 1 == queue.get_quota(pcpu_c, "@complexvalueshg")
assert 2 == queue.get_quota(pcpu_c, "@asdf")
def test_excl_parsing() -> None:
c = Complex("exclusive", "excl", "BOOL", "EXCL", True, True, "0", 1000)
assert c.default == Memory.value_of("0b")
assert c.parse("TRuE")
assert c.parse("true")
assert c.parse("1")
assert not c.parse("FALsE")
assert not c.parse("false")
assert not c.parse("0")
for expr in ["yes", "y", "SDF", None, ""]:
assert c.parse(expr) == expr
def test_memory_parsing() -> None:
c = Complex("m_mem_free", "mfree", "MEMORY", "<=", True, True, "0", 0)
assert c.parse("100") == Memory.value_of("100b")
assert c.parse("100g") == Memory.value_of("100g")
assert c.parse("100G") == Memory.value_of("100G")
assert c.parse("100g") != Memory.value_of("100G")
assert c.parse("42.123t") == Memory.value_of("42.123t")
assert c.name == "m_mem_free"
assert c.shortcut == "mfree"
for expr in ["blah.", "10gigs", None, ""]:
assert c.parse(expr) == expr
def run_test(
ctype: str,
node_lic: Optional[str],
hg_lic: str,
q_default_lic: str,
complex_default: Optional[str],
) -> SchedulerNode:
def cast(x: Optional[str]) -> Optional[str]:
if x is None:
return None
if ctype == "CSTRING":
return x.lower()
return x
node_res = {}
if node_lic is not None:
node_res["lic"] = cast(node_lic)
node_res["l"] = cast(node_lic)
node = SchedulerNode("tux", node_res)
ge_env = common_ge_env()
q = ge_env.queues["hpc.q"]
complex_default_str = (
"NONE" if complex_default is None else str(complex_default)
)
ge_env.complexes["lic"] = Complex(
"lic", "l", ctype, "<=", True, True, complex_default_str, 0
)
q.complex_values[None] = {"lic": cast(q_default_lic)}
q.complex_values["@hpc.q"] = {"lic": cast(hg_lic)}
assert node.available.get("lic") == node_lic
process_quotas(node, ge_env.complexes, ["@hpc.q"], [q])
return node
from hpc.autoscale.node.nodehistory import NullNodeHistory
from hpc.autoscale.results import DefaultContextHandler, register_result_handler
from hpc.autoscale.util import partition, partition_single
from gridengine import autoscaler
from gridengine.allocation_rules import FillUp, FixedProcesses, RoundRobin
from gridengine.complex import Complex
from gridengine.environment import GridEngineEnvironment
from gridengine.hostgroup import Hostgroup
from gridengine.parallel_environments import new_parallel_environment as new_pe
from gridengine.qbin import QBinImpl
from gridengine.queue import new_gequeue
from gridengine.scheduler import GridEngineScheduler
from gridengine_test import mock_driver
SLOTS_COMPLEX = Complex("slots", "s", "INT", "<=", True, True, "1", 1000)
MFREE_COMPLEX = Complex("m_mem_free", "mfree", "MEMORY", "<=", True, True, "0",
0)
EXCL_COMPLEX = Complex("exclusive", "excl", "BOOL", "EXCL", True, True, "0",
1000)
CONTEXT = DefaultContextHandler("[default]")
def setup_module() -> None:
SchedulerNode.ignore_hostnames = True
hpclogging.initialize_logging(mock_config(None))
register_result_handler(CONTEXT)
def test_non_exclusive_htc_arrays() -> None:
# ask for exactly the available count 10