def read_queues(
autoscale_config: Dict,
scheduler: "GridEngineScheduler",
pes: Dict[str, "ParallelEnvironment"],
hostgroups: List[Hostgroup],
complexes: Dict[str, "Complex"],
qbin: QBin,
) -> Dict[str, GridEngineQueue]:
queues = {}
qnames = qbin.qconf(["-sql"]).split()
logging.debug("Found %d queues: %s", len(qnames), " ".join(qnames))
autoscale_queues_config = autoscale_config.get("gridengine", {}).get("queues", {})
unbound_hostgroups = partition_single(hostgroups, lambda h: h.name)
for qname in qnames:
lines = qbin.qconf(["-sq", qname]).splitlines()
queue_config = parse_ge_config(lines)
autoscale_enabled = autoscale_queues_config.get(queue_config["qname"], {}).get(
"autoscale_enabled", True
)
expr = queue_config.get("complex_values", "NONE")
complex_values = parse_queue_complex_values(expr, complexes, qname)
queues[qname] = GridEngineQueue(
queue_config,
scheduler,
pes,
unbound_hostgroups,
complex_values,
autoscale_enabled,
)
return queues
def read_parallel_environments(
autoscale_config: Dict,
qbin: QBin,
) -> Dict[str, "ParallelEnvironment"]:
parallel_envs = {}
pe_config = autoscale_config.get("gridengine", {}).get("pes", {})
pe_names = qbin.qconf(["-spl"]).splitlines()
for pe_name in pe_names:
pe_name = pe_name.strip()
lines = qbin.qconf(["-sp", pe_name]).splitlines(False)
pe = parse_ge_config(lines)
req_key = "requires_placement_groups"
if req_key in pe_config.get(pe_name, {}):
logging.warning(
"Overriding placement group behavior for PE %s with %s",
pe_name,
pe_config[pe_name][req_key],
)
pe[req_key] = pe_config[pe_name][req_key]
parallel_envs[pe_name] = ParallelEnvironment(pe)
return parallel_envs
def read_hostgroups(autoscale_config: Dict, qbin: QBin) -> List[Hostgroup]:
# map each host (lowercase) to its set of hostgroups
ret: List[Hostgroup] = []
hg_config = autoscale_config.get("gridengine", {}).get("hostgroups", {})
for hg_name in qbin.qconf(["-shgrpl"]).split():
members = qbin.qconf(["-shgrp_resolved", hg_name]).split()
members = [h.split(".")[0].lower() for h in members]
constraints = hg_config.get(hg_name, {}).get("constraints", []) or []
if not isinstance(constraints, list):
constraints = [constraints]
parsed_constraints = constraintslib.get_constraints(constraints)
hostgroup = Hostgroup(hg_name, parsed_constraints, members)
ret.append(hostgroup)
return ret
def validate_queue_has_hosts(queue: GridEngineQueue, qbin: QBin,
warn_function: WarnFunction) -> bool:
# ensure that there is at least one host for each queue.
empty_hostgroups = []
for hostgroup in queue.hostlist:
if hostgroup == "NONE":
continue
if not hostgroup.startswith("@"):
# explicit hostnames are obviously
return True
if qbin.qconf(["-shgrp_resolved", hostgroup]).strip():
return True
empty_hostgroups.append(hostgroup)
warn_function(
"Queue %s has no hosts assigned to any of its hostgroups and you will not be able to submit to it.",
queue.qname,
)
warn_function(" Empty hostgroups = %s", " ".join(empty_hostgroups))
return False
def read_complexes(autoscale_config: Dict, qbin: QBin) -> Dict[str, "Complex"]:
complex_lines = qbin.qconf(["-sc"]).splitlines()
return _parse_complexes(autoscale_config, complex_lines)
def read_scheduler(qbin: QBin) -> GridEngineScheduler:
lines = qbin.qconf(["-ssconf"]).splitlines()
sched_config = util.parse_ge_config(lines)
return GridEngineScheduler(sched_config)