def analyze(config: Dict, job_id: str, wide: bool = False) -> None: if not wide: try: _, columns_str = os.popen("stty size", "r").read().split() except Exception: columns_str = "120" columns = int(columns_str) else: columns = 120 ctx = DefaultContextHandler("[demand-cli]") register_result_handler(ctx) ge_env = environment.from_qconf(config) ge_driver = autoscaler.new_driver(config, ge_env) config = ge_driver.preprocess_config(config) autoscaler.calculate_demand(config, ge_env, ge_driver, ctx) key = "[job {}]".format(job_id) results = ctx.by_context[key] for result in results: if isinstance(result, (EarlyBailoutResult, MatchResult)) and result: continue if isinstance(result, HostgroupConstraint) and not result: continue if wide: print(result.message) else: print(result.message[:columns])
def autoscale( self, config: Dict, output_columns: Optional[List[str]], output_format: OutputFormat, dry_run: bool = False, long: bool = False, ) -> None: """End-to-end autoscale process, including creation, deletion and joining of nodes.""" output_columns = output_columns or self._get_default_output_columns( config) ctx_handler = self._ctx_handler(config) register_result_handler(ctx_handler) driver = self._driver(config) driver.initialize() config = driver.preprocess_config(config) logging.debug("Driver = %s", driver) return autoscale_hpcpack(config, ctx_handler=ctx_handler, dry_run=dry_run)
def test_choice_ordering() -> None: bindings = MockClusterBinding() bindings.add_nodearray("array-a", {"nodetype": "A"}) bindings.add_bucket("array-a", "Standard_F4", 10, 10) bindings.add_nodearray("array-b", {"nodetype": "B"}) bindings.add_bucket("array-b", "Standard_F4s", 10, 10) register_result_handler(DefaultContextHandler("[test_or_ordering]")) for ordering in [["A", "B"], ["B", "A"]]: node_mgr = _node_mgr(bindings) hi, lo = node_mgr.get_buckets() if hi.resources["nodetype"] != ordering[0]: hi, lo = lo, hi assert hi.available_count == 10 assert lo.available_count == 10 result = node_mgr.allocate( { "nodetype": ordering, "exclusive": True, }, node_count=15, # noqa: E231 ) assert hi.available_count == 0 assert lo.available_count == 5 assert result by_array = partition(result.nodes, lambda n: n.resources["nodetype"]) assert len(by_array[ordering[0]]) == 10 assert len(by_array[ordering[1]]) == 5
def test_excl_colocated_packing_bug() -> None: def n() -> NodeManager: return _node_mgr(_bindings()) # assert [] == node_mgr.get_nodes() # result = node_mgr.allocate({"node.nodearray": "htc", "ncpus": 1, "exclusive": True}, slot_count=2_000_000, all_or_nothing=True) # assert not result, str(result) # assert [] == node_mgr.get_nodes() # assert [] == node_mgr.new_nodes register_result_handler(DefaultContextHandler("[ttt]")) result = n().allocate( { "node.nodearray": "htc", "ncpus": 1, "exclusive": True }, slot_count=10, all_or_nothing=True, ) assert len(result.nodes) == 3, len(result.nodes) result = n().allocate( { "node.nodearray": "htc", "ncpus": 1, "exclusive": True, "node.vm_size": "Standard_F4s", }, slot_count=19, all_or_nothing=True, ) assert len(result.nodes) == 5, len(result.nodes) result = n().allocate( { "node.nodearray": "htc", "ncpus": 1, "exclusive": True, "node.vm_size": "Standard_F4s", }, node_count=101, all_or_nothing=True, ) assert not result, result
def invoke(*args: Any, **kwargs: Any) -> Any: handler = register_result_handler( DefaultContextHandler("[{}]".format(func.__name__))) if "handler" in inspect.signature(func).parameters: kwargs["handler"] = handler ret = func(*args, **kwargs) unregister_result_handler(handler) return ret
def demand( config: Dict, jobs: Optional[str] = None, scheduler_nodes: Optional[str] = None, output_columns: Optional[List[str]] = None, output_format: Optional[str] = None, ) -> None: """Runs autoscale in dry run mode to see the demand for new nodes""" logging.debug("Begin demand") ctx = DefaultContextHandler("[demand-cli]") register_result_handler(ctx) ge_env = environment.from_qconf(config) ge_driver = autoscaler.new_driver(config, ge_env) config = ge_driver.preprocess_config(config) demand_calc = autoscaler.calculate_demand(config, ge_env, ge_driver, ctx) demand_result = demand_calc.finish() autoscaler.print_demand(config, demand_result, output_columns, output_format) logging.debug("End demand")
def autoscale( config: Dict, output_columns: Optional[List[str]] = None, output_format: Optional[str] = None, ) -> None: """Runs actual autoscale process""" logging.debug("Begin autoscale") ctx_handler = register_result_handler(DefaultContextHandler("[initialization]")) if output_columns: config["output_columns"] = output_columns if output_format: config["output_format"] = output_format autoscaler.autoscale_grid_engine(config, ctx_handler=ctx_handler) logging.debug("End autoscale")
def main() -> int: ctx_handler = register_result_handler( DefaultContextHandler("[initialization]")) parser = ArgumentParser() parser.add_argument("-c", "--config", help="Path to autoscale config.", required=True) args = parser.parse_args() config_path = os.path.expanduser(args.config) if not os.path.exists(config_path): print("{} does not exist.".format(config_path), file=sys.stderr) return 1 config = json_load(config_path) autoscale_pbspro(config, ctx_handler=ctx_handler) return _exit_code
def setup_function(function) -> None: resultslib.register_result_handler( resultslib.DefaultContextHandler("[{}]".format(function.__name__)) )
def setup_module() -> None: SchedulerNode.ignore_hostnames = True hpclogging.initialize_logging(mock_config(None)) register_result_handler(CONTEXT)
def new_rest_client(config: Dict[str, Any]) -> HpcRestClient: hpcpack_config = config.get('hpcpack') or {} hpc_pem_file = hpcpack_config.get('pem') hn_hostname = hpcpack_config.get('hn_hostname') return HpcRestClient(config, pem=hpc_pem_file, hostname=hn_hostname) if __name__ == "__main__": config_file = "" if len(sys.argv) > 1: config_file = sys.argv[1] dry_run = False if len(sys.argv) > 2: dry_run = ci_in(sys.argv[2], ['true', 'dryrun']) ctx_handler = register_result_handler( DefaultContextHandler("[initialization]")) config = load_config(config_file) logging.initialize_logging(config) logging.info( "------------------------------------------------------------------------" ) if config["autoscale"]["start_enabled"]: autoscale_hpcpack(config, ctx_handler=ctx_handler, dry_run=dry_run) else: logging.info("Autoscaler is not enabled")
def setup_function(function) -> None: SchedulerNode.ignore_hostnames = True results.register_result_handler( results.DefaultContextHandler("[{}]".format(function.__name__)))
def test_mock_bindings(bindings: MockClusterBinding) -> None: ctx = register_result_handler(DefaultContextHandler("[test]")) hpc, htc = _node_mgr(bindings).get_buckets() if hpc.nodearray != "hpc": hpc, htc = htc, hpc assert hpc.nodearray == "hpc" assert htc.nodearray == "htc" assert hpc.family_available_count == 10 assert hpc.available_count == 10 assert hpc.family_available_count == 10 assert htc.family_available_count == 20 hpc.decrement(1) assert hpc.family_available_count == 9 assert htc.family_available_count == 20 hpc.commit() assert hpc.family_available_count == 9 assert htc.family_available_count == 18 hpc.increment(1) hpc.commit() assert hpc.family_available_count == 10 assert htc.family_available_count == 20 ctx.set_context("[failure]") nm = _node_mgr(bindings) b = MockClusterBinding() b.add_nodearray("haspgs", {}, max_placement_group_size=20) b.add_bucket( "haspgs", "Standard_F4", 100, 100, placement_groups=["pg0", "pg1"], ) # make sure we take the max_placement_group_size (20) into account # and that we have the non-pg and 2 pg buckets. nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 100 assert pg0.available_count == 20 assert pg1.available_count == 20 # let's add a node to pg0 (100 - 1, 20 - 1, 20) b.add_node("haspgs-pg0-1", "haspgs", "Standard_F4", placement_group="pg0") nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 99 assert pg0.available_count == 19 assert pg1.available_count == 20 # let's add a node to pg1 (100 - 2, 20 - 1, 20 - 1) b.add_node("haspgs-pg1-1", "haspgs", "Standard_F4", placement_group="pg1") nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 98 assert pg0.available_count == 19 assert pg1.available_count == 19 # let's add 90 htc nodes so that our pg available counts are floored # by the overall available_count for i in range(90): b.add_node("haspgs-{}".format(i + 1), "haspgs", "Standard_F4") nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 8 assert pg0.available_count == 8 assert pg1.available_count == 8 # lastly, add a nother node to a pg and see that all of avail go down b.add_node("haspgs-pg1-2", "haspgs", "Standard_F4", placement_group="pg1") nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 7 assert pg0.available_count == 7 assert pg1.available_count == 7