def test_minimum_space() -> None:
c = MinResourcePerNode("pcpus", 1)
assert 1 == c.minimum_space(SchedulerNode("", {"pcpus": 1}))
assert 2 == c.minimum_space(SchedulerNode("", {"pcpus": 2}))
snode = SchedulerNode("", {"pcpus": 2})
assert -1 == ExclusiveNode(assignment_id="1").minimum_space(snode)
snode.assign("1")
assert 0 == ExclusiveNode(assignment_id="1").minimum_space(snode)
def do_draw(self, data: Any) -> ht.VMSize:
import hypothesis.internal.conjecture.utils as d
idx = d.integer_range(data, 0, 1_000_000_000)
r = random.Random(idx)
def draw_value() -> Optional[Any]:
rtype_draw = r.randint(0, 4)
if rtype_draw == 0:
return r.randint(0, 100)
elif rtype_draw == 1:
return r.random() * 100
elif rtype_draw == 2:
def draw_letter():
return r.choice(string.ascii_letters)
return "".join([draw_letter() for n in range(r.randint(0, 100))])
elif rtype_draw == 3:
return r.random() < 0.5
else:
return None
hostname = "n-o-d-e_-{}".format(r.randint(1, 1000000))
resources: Dict[str, Optional[Any]] = {}
num_resources = r.randint(0, 10)
for n in range(num_resources):
rname = "res-{}".format(n)
resources[rname] = draw_value()
node = SchedulerNode(ht.Hostname(hostname), resources, "bucket-id-123")
for job_id in range(r.randint(0, 10)):
node.assign(str(job_id))
num_meta = r.randint(0, 10)
for n in range(num_meta):
mname = "meta-{}".format(n)
node.metadata[mname] = draw_value()
for rname, rvalue in node.resources.items():
if r.random() > 0.5:
if isinstance(rvalue, int):
node.available[rname] = rvalue - r.randint(0, rvalue + 1)
elif isinstance(rvalue, float):
node.available[rname] = rvalue * r.random()
elif isinstance(rvalue, bool):
node.available[rname] = rvalue ^ (r.random() < 0.5)
elif rvalue is None:
# in theory you can change a null resource
# into an actual value as available
if r.random() < 0.25:
node.available[rname] = draw_value()
return node
def _print_demand(output_format: OutputFormat) -> str:
stream = io.StringIO()
node = SchedulerNode("tux", {"ncpus": 2, "mem": Memory.value_of("1.0g")})
node.available["ncpus"] = 1
node.assign("11")
node.assign("12")
result = DemandResult([], [node], [], [])
print_demand(
["hostname", "job_ids", "ncpus", "*ncpus", "mem"],
result,
stream=stream,
output_format=output_format,
)
return stream.getvalue()
def test_job_excl() -> None:
s = SchedulerNode("")
# typical exclusive behavior - one task per job per node
job_excl = get_constraint({"exclusive": True})
assert job_excl.job_exclusive
assert isinstance(job_excl, ExclusiveNode)
assert job_excl.satisfied_by_node(s)
assert -1 == job_excl.minimum_space(s)
assert job_excl.do_decrement(s)
s.assign("1")
job_excl.assignment_id = "1"
# can't put the same jobid on the same node twice
assert not job_excl.satisfied_by_node(s)
assert not job_excl.do_decrement(s)
assert s.closed
assert 0 == job_excl.minimum_space(s)
def test_task_excl() -> None:
s = SchedulerNode("")
# now to test tack exclusive, where multiple tasks from the same
# job can run on the same machine
task_excl = get_constraint({"exclusive_task": True})
assert not task_excl.job_exclusive
assert isinstance(task_excl, ExclusiveNode)
assert task_excl.satisfied_by_node(s)
assert -1 == task_excl.minimum_space(s)
assert task_excl.do_decrement(s)
s.assign("1")
task_excl.assignment_id = "1"
assert task_excl.satisfied_by_node(s)
assert task_excl.do_decrement(s)
assert s.closed
assert -1 == task_excl.minimum_space(s)
def parse_scheduler_node(
ndict: Dict[str, Any],
resource_definitions: Dict[str,
PBSProResourceDefinition]) -> SchedulerNode:
"""
Implementation of parsing a single scheduler node.
"""
parser = get_pbspro_parser()
hostname = ndict["name"]
res_avail = parser.parse_resources_available(ndict, filter_is_host=True)
res_assigned = parser.parse_resources_assigned(ndict, filter_is_host=True)
node = SchedulerNode(hostname, res_avail)
jobs_expr = ndict.get("jobs", "")
state = ndict.get("state") or ""
if state == "free" and jobs_expr.strip():
state = "partially-free"
node.metadata["pbs_state"] = state
if "down" in state:
node.marked_for_deletion = True
node.metadata["last_state_change_time"] = ndict.get(
"last_state_change_time", "")
for tok in jobs_expr.split(","):
tok = tok.strip()
if not tok:
continue
job_id_full, sub_job_id = tok.rsplit("/", 1)
sched_host = ""
if "." in job_id_full:
job_id, sched_host = job_id_full.split(".", 1)
else:
job_id = job_id_full
node.assign(job_id)
if "job_ids_long" not in node.metadata:
node.metadata["job_ids_long"] = [job_id_full]
elif job_id_full not in node.metadata["job_ids_long"]:
node.metadata["job_ids_long"].append(job_id_full)
for res_name, value in res_assigned.items():
resource = resource_definitions.get(res_name)
if not resource or not resource.is_host:
continue
if resource.is_consumable:
if res_name in node.available:
node.available[res_name] -= value
else:
logging.warning(
"%s was not defined under resources_available, but was " +
"defined under resources_assigned for %s. Setting available to assigned.",
res_name,
node,
)
node.available[res_name] = value
if "exclusive" in node.metadata["pbs_state"]:
node.closed = True
return node
