def analyze(config: Dict, job_id: str, wide: bool = False) -> None:
if not wide:
try:
_, columns_str = os.popen("stty size", "r").read().split()
except Exception:
columns_str = "120"
columns = int(columns_str)
else:
columns = 120
ctx = DefaultContextHandler("[demand-cli]")
register_result_handler(ctx)
ge_env = environment.from_qconf(config)
ge_driver = autoscaler.new_driver(config, ge_env)
config = ge_driver.preprocess_config(config)
autoscaler.calculate_demand(config, ge_env, ge_driver, ctx)
key = "[job {}]".format(job_id)
results = ctx.by_context[key]
for result in results:
if isinstance(result, (EarlyBailoutResult, MatchResult)) and result:
continue
if isinstance(result, HostgroupConstraint) and not result:
continue
if wide:
print(result.message)
else:
print(result.message[:columns])
def autoscale(
self,
config: Dict,
output_columns: Optional[List[str]],
output_format: OutputFormat,
dry_run: bool = False,
long: bool = False,
) -> None:
"""End-to-end autoscale process, including creation, deletion and joining of nodes."""
output_columns = output_columns or self._get_default_output_columns(
config)
ctx_handler = self._ctx_handler(config)
register_result_handler(ctx_handler)
driver = self._driver(config)
driver.initialize()
config = driver.preprocess_config(config)
logging.debug("Driver = %s", driver)
return autoscale_hpcpack(config,
ctx_handler=ctx_handler,
dry_run=dry_run)
def test_choice_ordering() -> None:
bindings = MockClusterBinding()
bindings.add_nodearray("array-a", {"nodetype": "A"})
bindings.add_bucket("array-a", "Standard_F4", 10, 10)
bindings.add_nodearray("array-b", {"nodetype": "B"})
bindings.add_bucket("array-b", "Standard_F4s", 10, 10)
register_result_handler(DefaultContextHandler("[test_or_ordering]"))
for ordering in [["A", "B"], ["B", "A"]]:
node_mgr = _node_mgr(bindings)
hi, lo = node_mgr.get_buckets()
if hi.resources["nodetype"] != ordering[0]:
hi, lo = lo, hi
assert hi.available_count == 10
assert lo.available_count == 10
result = node_mgr.allocate(
{
"nodetype": ordering,
"exclusive": True,
},
node_count=15, # noqa: E231
)
assert hi.available_count == 0
assert lo.available_count == 5
assert result
by_array = partition(result.nodes, lambda n: n.resources["nodetype"])
assert len(by_array[ordering[0]]) == 10
assert len(by_array[ordering[1]]) == 5
def test_excl_colocated_packing_bug() -> None:
def n() -> NodeManager:
return _node_mgr(_bindings())
# assert [] == node_mgr.get_nodes()
# result = node_mgr.allocate({"node.nodearray": "htc", "ncpus": 1, "exclusive": True}, slot_count=2_000_000, all_or_nothing=True)
# assert not result, str(result)
# assert [] == node_mgr.get_nodes()
# assert [] == node_mgr.new_nodes
register_result_handler(DefaultContextHandler("[ttt]"))
result = n().allocate(
{
"node.nodearray": "htc",
"ncpus": 1,
"exclusive": True
},
slot_count=10,
all_or_nothing=True,
)
assert len(result.nodes) == 3, len(result.nodes)
result = n().allocate(
{
"node.nodearray": "htc",
"ncpus": 1,
"exclusive": True,
"node.vm_size": "Standard_F4s",
},
slot_count=19,
all_or_nothing=True,
)
assert len(result.nodes) == 5, len(result.nodes)
result = n().allocate(
{
"node.nodearray": "htc",
"ncpus": 1,
"exclusive": True,
"node.vm_size": "Standard_F4s",
},
node_count=101,
all_or_nothing=True,
)
assert not result, result
def invoke(*args: Any, **kwargs: Any) -> Any:
handler = register_result_handler(
DefaultContextHandler("[{}]".format(func.__name__)))
if "handler" in inspect.signature(func).parameters:
kwargs["handler"] = handler
ret = func(*args, **kwargs)
unregister_result_handler(handler)
return ret
def demand(
config: Dict,
jobs: Optional[str] = None,
scheduler_nodes: Optional[str] = None,
output_columns: Optional[List[str]] = None,
output_format: Optional[str] = None,
) -> None:
"""Runs autoscale in dry run mode to see the demand for new nodes"""
logging.debug("Begin demand")
ctx = DefaultContextHandler("[demand-cli]")
register_result_handler(ctx)
ge_env = environment.from_qconf(config)
ge_driver = autoscaler.new_driver(config, ge_env)
config = ge_driver.preprocess_config(config)
demand_calc = autoscaler.calculate_demand(config, ge_env, ge_driver, ctx)
demand_result = demand_calc.finish()
autoscaler.print_demand(config, demand_result, output_columns, output_format)
logging.debug("End demand")
def autoscale(
config: Dict,
output_columns: Optional[List[str]] = None,
output_format: Optional[str] = None,
) -> None:
"""Runs actual autoscale process"""
logging.debug("Begin autoscale")
ctx_handler = register_result_handler(DefaultContextHandler("[initialization]"))
if output_columns:
config["output_columns"] = output_columns
if output_format:
config["output_format"] = output_format
autoscaler.autoscale_grid_engine(config, ctx_handler=ctx_handler)
logging.debug("End autoscale")
def main() -> int:
ctx_handler = register_result_handler(
DefaultContextHandler("[initialization]"))
parser = ArgumentParser()
parser.add_argument("-c",
"--config",
help="Path to autoscale config.",
required=True)
args = parser.parse_args()
config_path = os.path.expanduser(args.config)
if not os.path.exists(config_path):
print("{} does not exist.".format(config_path), file=sys.stderr)
return 1
config = json_load(config_path)
autoscale_pbspro(config, ctx_handler=ctx_handler)
return _exit_code
def setup_function(function) -> None:
resultslib.register_result_handler(
resultslib.DefaultContextHandler("[{}]".format(function.__name__))
)
def setup_module() -> None:
SchedulerNode.ignore_hostnames = True
hpclogging.initialize_logging(mock_config(None))
register_result_handler(CONTEXT)
def new_rest_client(config: Dict[str, Any]) -> HpcRestClient:
hpcpack_config = config.get('hpcpack') or {}
hpc_pem_file = hpcpack_config.get('pem')
hn_hostname = hpcpack_config.get('hn_hostname')
return HpcRestClient(config, pem=hpc_pem_file, hostname=hn_hostname)
if __name__ == "__main__":
config_file = ""
if len(sys.argv) > 1:
config_file = sys.argv[1]
dry_run = False
if len(sys.argv) > 2:
dry_run = ci_in(sys.argv[2], ['true', 'dryrun'])
ctx_handler = register_result_handler(
DefaultContextHandler("[initialization]"))
config = load_config(config_file)
logging.initialize_logging(config)
logging.info(
"------------------------------------------------------------------------"
)
if config["autoscale"]["start_enabled"]:
autoscale_hpcpack(config, ctx_handler=ctx_handler, dry_run=dry_run)
else:
logging.info("Autoscaler is not enabled")
def setup_function(function) -> None:
SchedulerNode.ignore_hostnames = True
results.register_result_handler(
results.DefaultContextHandler("[{}]".format(function.__name__)))
def test_mock_bindings(bindings: MockClusterBinding) -> None:
ctx = register_result_handler(DefaultContextHandler("[test]"))
hpc, htc = _node_mgr(bindings).get_buckets()
if hpc.nodearray != "hpc":
hpc, htc = htc, hpc
assert hpc.nodearray == "hpc"
assert htc.nodearray == "htc"
assert hpc.family_available_count == 10
assert hpc.available_count == 10
assert hpc.family_available_count == 10
assert htc.family_available_count == 20
hpc.decrement(1)
assert hpc.family_available_count == 9
assert htc.family_available_count == 20
hpc.commit()
assert hpc.family_available_count == 9
assert htc.family_available_count == 18
hpc.increment(1)
hpc.commit()
assert hpc.family_available_count == 10
assert htc.family_available_count == 20
ctx.set_context("[failure]")
nm = _node_mgr(bindings)
b = MockClusterBinding()
b.add_nodearray("haspgs", {}, max_placement_group_size=20)
b.add_bucket(
"haspgs",
"Standard_F4",
100,
100,
placement_groups=["pg0", "pg1"],
)
# make sure we take the max_placement_group_size (20) into account
# and that we have the non-pg and 2 pg buckets.
nm = _node_mgr(b)
no_pg, pg0, pg1 = sorted(nm.get_buckets(),
key=lambda b: b.placement_group or "")
assert no_pg.available_count == 100
assert pg0.available_count == 20
assert pg1.available_count == 20
# let's add a node to pg0 (100 - 1, 20 - 1, 20)
b.add_node("haspgs-pg0-1", "haspgs", "Standard_F4", placement_group="pg0")
nm = _node_mgr(b)
no_pg, pg0, pg1 = sorted(nm.get_buckets(),
key=lambda b: b.placement_group or "")
assert no_pg.available_count == 99
assert pg0.available_count == 19
assert pg1.available_count == 20
# let's add a node to pg1 (100 - 2, 20 - 1, 20 - 1)
b.add_node("haspgs-pg1-1", "haspgs", "Standard_F4", placement_group="pg1")
nm = _node_mgr(b)
no_pg, pg0, pg1 = sorted(nm.get_buckets(),
key=lambda b: b.placement_group or "")
assert no_pg.available_count == 98
assert pg0.available_count == 19
assert pg1.available_count == 19
# let's add 90 htc nodes so that our pg available counts are floored
# by the overall available_count
for i in range(90):
b.add_node("haspgs-{}".format(i + 1), "haspgs", "Standard_F4")
nm = _node_mgr(b)
no_pg, pg0, pg1 = sorted(nm.get_buckets(),
key=lambda b: b.placement_group or "")
assert no_pg.available_count == 8
assert pg0.available_count == 8
assert pg1.available_count == 8
# lastly, add a nother node to a pg and see that all of avail go down
b.add_node("haspgs-pg1-2", "haspgs", "Standard_F4", placement_group="pg1")
nm = _node_mgr(b)
no_pg, pg0, pg1 = sorted(nm.get_buckets(),
key=lambda b: b.placement_group or "")
assert no_pg.available_count == 7
assert pg0.available_count == 7
assert pg1.available_count == 7
