def _setup_shell_locals(self, config: Dict) -> Dict: ctx = DefaultContextHandler("[interactive-readonly]") driver = self._driver(config) dcalc, jobs_list = self._demand_calc(config, driver) nodes_list = dcalc.node_mgr.get_nodes() for node in nodes_list: node.shellify() nodes = partition_single(nodes_list, lambda n: n.name) nodes.update( partition_single([x for x in nodes_list if x.hostname], lambda n: n.hostname)) jobs: Dict[str, Any] try: jobs = partition_single(jobs_list, lambda j: j.name) except Exception: jobs = partition(jobs_list, lambda j: j.name) return { "config": config, "cli": self, "ctx": ctx, "demand_calc": dcalc, "node_mgr": dcalc.node_mgr, "jobs": ShellDict(jobs), "nodes": ShellDict(nodes), }
def _setup_shell_locals(self, config: Dict) -> Dict: """ Provides read only interactive shell. type hpcpackhelp() in the shell for more information """ ctx = DefaultContextHandler("[interactive-readonly]") def hpcpackhelp() -> None: print( "config - dict representing autoscale configuration." ) print( "cli - object representing the CLI commands") print( "node_mgr - ScaleLib NodeManager - interacts with CycleCloud for all node related" + " activities - creation, deletion, limits, buckets etc." ) print("hpcpackhelp - This help function") shell_locals = { "config": config, "cli": self, "ctx": ctx, "node_mgr": new_node_manager(config), "hpcpackhelp": hpcpackhelp, } return shell_locals
def test_choice_ordering() -> None: bindings = MockClusterBinding() bindings.add_nodearray("array-a", {"nodetype": "A"}) bindings.add_bucket("array-a", "Standard_F4", 10, 10) bindings.add_nodearray("array-b", {"nodetype": "B"}) bindings.add_bucket("array-b", "Standard_F4s", 10, 10) register_result_handler(DefaultContextHandler("[test_or_ordering]")) for ordering in [["A", "B"], ["B", "A"]]: node_mgr = _node_mgr(bindings) hi, lo = node_mgr.get_buckets() if hi.resources["nodetype"] != ordering[0]: hi, lo = lo, hi assert hi.available_count == 10 assert lo.available_count == 10 result = node_mgr.allocate( { "nodetype": ordering, "exclusive": True, }, node_count=15, # noqa: E231 ) assert hi.available_count == 0 assert lo.available_count == 5 assert result by_array = partition(result.nodes, lambda n: n.resources["nodetype"]) assert len(by_array[ordering[0]]) == 10 assert len(by_array[ordering[1]]) == 5
def analyze(config: Dict, job_id: str, wide: bool = False) -> None: if not wide: try: _, columns_str = os.popen("stty size", "r").read().split() except Exception: columns_str = "120" columns = int(columns_str) else: columns = 120 ctx = DefaultContextHandler("[demand-cli]") register_result_handler(ctx) ge_env = environment.from_qconf(config) ge_driver = autoscaler.new_driver(config, ge_env) config = ge_driver.preprocess_config(config) autoscaler.calculate_demand(config, ge_env, ge_driver, ctx) key = "[job {}]".format(job_id) results = ctx.by_context[key] for result in results: if isinstance(result, (EarlyBailoutResult, MatchResult)) and result: continue if isinstance(result, HostgroupConstraint) and not result: continue if wide: print(result.message) else: print(result.message[:columns])
def invoke(*args: Any, **kwargs: Any) -> Any: handler = register_result_handler( DefaultContextHandler("[{}]".format(func.__name__))) if "handler" in inspect.signature(func).parameters: kwargs["handler"] = handler ret = func(*args, **kwargs) unregister_result_handler(handler) return ret
def autoscale( config: Dict, output_columns: Optional[List[str]] = None, output_format: Optional[str] = None, ) -> None: """Runs actual autoscale process""" logging.debug("Begin autoscale") ctx_handler = register_result_handler(DefaultContextHandler("[initialization]")) if output_columns: config["output_columns"] = output_columns if output_format: config["output_format"] = output_format autoscaler.autoscale_grid_engine(config, ctx_handler=ctx_handler) logging.debug("End autoscale")
def test_excl_colocated_packing_bug() -> None: def n() -> NodeManager: return _node_mgr(_bindings()) # assert [] == node_mgr.get_nodes() # result = node_mgr.allocate({"node.nodearray": "htc", "ncpus": 1, "exclusive": True}, slot_count=2_000_000, all_or_nothing=True) # assert not result, str(result) # assert [] == node_mgr.get_nodes() # assert [] == node_mgr.new_nodes register_result_handler(DefaultContextHandler("[ttt]")) result = n().allocate( { "node.nodearray": "htc", "ncpus": 1, "exclusive": True }, slot_count=10, all_or_nothing=True, ) assert len(result.nodes) == 3, len(result.nodes) result = n().allocate( { "node.nodearray": "htc", "ncpus": 1, "exclusive": True, "node.vm_size": "Standard_F4s", }, slot_count=19, all_or_nothing=True, ) assert len(result.nodes) == 5, len(result.nodes) result = n().allocate( { "node.nodearray": "htc", "ncpus": 1, "exclusive": True, "node.vm_size": "Standard_F4s", }, node_count=101, all_or_nothing=True, ) assert not result, result
def demand( config: Dict, jobs: Optional[str] = None, scheduler_nodes: Optional[str] = None, output_columns: Optional[List[str]] = None, output_format: Optional[str] = None, ) -> None: """Runs autoscale in dry run mode to see the demand for new nodes""" logging.debug("Begin demand") ctx = DefaultContextHandler("[demand-cli]") register_result_handler(ctx) ge_env = environment.from_qconf(config) ge_driver = autoscaler.new_driver(config, ge_env) config = ge_driver.preprocess_config(config) demand_calc = autoscaler.calculate_demand(config, ge_env, ge_driver, ctx) demand_result = demand_calc.finish() autoscaler.print_demand(config, demand_result, output_columns, output_format) logging.debug("End demand")
def main() -> int: ctx_handler = register_result_handler( DefaultContextHandler("[initialization]")) parser = ArgumentParser() parser.add_argument("-c", "--config", help="Path to autoscale config.", required=True) args = parser.parse_args() config_path = os.path.expanduser(args.config) if not os.path.exists(config_path): print("{} does not exist.".format(config_path), file=sys.stderr) return 1 config = json_load(config_path) autoscale_pbspro(config, ctx_handler=ctx_handler) return _exit_code
from gridengine.allocation_rules import FillUp, FixedProcesses, RoundRobin from gridengine.complex import Complex from gridengine.environment import GridEngineEnvironment from gridengine.hostgroup import Hostgroup from gridengine.parallel_environments import new_parallel_environment as new_pe from gridengine.qbin import QBinImpl from gridengine.queue import new_gequeue from gridengine.scheduler import GridEngineScheduler from gridengine_test import mock_driver SLOTS_COMPLEX = Complex("slots", "s", "INT", "<=", True, True, "1", 1000) MFREE_COMPLEX = Complex("m_mem_free", "mfree", "MEMORY", "<=", True, True, "0", 0) EXCL_COMPLEX = Complex("exclusive", "excl", "BOOL", "EXCL", True, True, "0", 1000) CONTEXT = DefaultContextHandler("[default]") def setup_module() -> None: SchedulerNode.ignore_hostnames = True hpclogging.initialize_logging(mock_config(None)) register_result_handler(CONTEXT) def test_non_exclusive_htc_arrays() -> None: # ask for exactly the available count 10 common_cluster_test(["-l nodearray=htc -t 1-40 -q htc.q sleep.sh"], htc=10) # ask for more than 10, hit limit common_cluster_test(["-t 1-44 -q htc.q sleep.sh"], htc=10)
def test_mock_bindings(bindings: MockClusterBinding) -> None: ctx = register_result_handler(DefaultContextHandler("[test]")) hpc, htc = _node_mgr(bindings).get_buckets() if hpc.nodearray != "hpc": hpc, htc = htc, hpc assert hpc.nodearray == "hpc" assert htc.nodearray == "htc" assert hpc.family_available_count == 10 assert hpc.available_count == 10 assert hpc.family_available_count == 10 assert htc.family_available_count == 20 hpc.decrement(1) assert hpc.family_available_count == 9 assert htc.family_available_count == 20 hpc.commit() assert hpc.family_available_count == 9 assert htc.family_available_count == 18 hpc.increment(1) hpc.commit() assert hpc.family_available_count == 10 assert htc.family_available_count == 20 ctx.set_context("[failure]") nm = _node_mgr(bindings) b = MockClusterBinding() b.add_nodearray("haspgs", {}, max_placement_group_size=20) b.add_bucket( "haspgs", "Standard_F4", 100, 100, placement_groups=["pg0", "pg1"], ) # make sure we take the max_placement_group_size (20) into account # and that we have the non-pg and 2 pg buckets. nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 100 assert pg0.available_count == 20 assert pg1.available_count == 20 # let's add a node to pg0 (100 - 1, 20 - 1, 20) b.add_node("haspgs-pg0-1", "haspgs", "Standard_F4", placement_group="pg0") nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 99 assert pg0.available_count == 19 assert pg1.available_count == 20 # let's add a node to pg1 (100 - 2, 20 - 1, 20 - 1) b.add_node("haspgs-pg1-1", "haspgs", "Standard_F4", placement_group="pg1") nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 98 assert pg0.available_count == 19 assert pg1.available_count == 19 # let's add 90 htc nodes so that our pg available counts are floored # by the overall available_count for i in range(90): b.add_node("haspgs-{}".format(i + 1), "haspgs", "Standard_F4") nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 8 assert pg0.available_count == 8 assert pg1.available_count == 8 # lastly, add a nother node to a pg and see that all of avail go down b.add_node("haspgs-pg1-2", "haspgs", "Standard_F4", placement_group="pg1") nm = _node_mgr(b) no_pg, pg0, pg1 = sorted(nm.get_buckets(), key=lambda b: b.placement_group or "") assert no_pg.available_count == 7 assert pg0.available_count == 7 assert pg1.available_count == 7
def use_result_handler() -> None: # use request_id in context request_id = str(uuid4()) handler = DefaultContextHandler("") handler.set_context("[{}]".format(request_id))
def autoscale_hpcpack( config: Dict[str, Any], ctx_handler: DefaultContextHandler = None, hpcpack_rest_client: Optional[HpcRestClient] = None, dry_run: bool = False, ) -> None: if not hpcpack_rest_client: hpcpack_rest_client = new_rest_client(config) if ctx_handler: ctx_handler.set_context("[Sync-Status]") autoscale_config = config.get("autoscale") or {} # Load history info idle_timeout_seconds: int = autoscale_config.get("idle_timeout") or 600 provisioning_timeout_seconds = autoscale_config.get("boot_timeout") or 1500 statefile = autoscale_config.get( "statefile") or "C:\\cycle\\jetpack\\config\\autoscaler_state.txt" archivefile = autoscale_config.get( "archivefile") or "C:\\cycle\\jetpack\\config\\autoscaler_archive.txt" node_history = HpcNodeHistory( statefile=statefile, archivefile=archivefile, provisioning_timeout=provisioning_timeout_seconds, idle_timeout=idle_timeout_seconds) logging.info("Synchronizing the nodes between Cycle cloud and HPC Pack") # Initialize data of History info, cc nodes, HPC Pack nodes, HPC grow decisions # Get node list from Cycle Cloud def nodes_state_key(n: Node) -> Tuple[int, str, int]: try: state_pri = 1 if n.state == 'Deallocated': state_pri = 2 elif n.state == 'Stopping': state_pri = 3 elif n.state == 'Terminating': state_pri = 4 name, index = n.name.rsplit("-", 1) return (state_pri, name, int(index)) except Exception: return (state_pri, n.name, 0) node_mgr: NodeManager = new_node_manager(config) for b in node_mgr.get_buckets(): b.nodes.sort(key=nodes_state_key) cc_nodes: List[Node] = node_mgr.get_nodes() cc_nodes_by_id = partition_single(cc_nodes, func=lambda n: n.delayed_node_id.node_id) # Get compute node list and grow decision from HPC Pack hpc_node_groups = hpcpack_rest_client.list_node_groups() grow_decisions = hpcpack_rest_client.get_grow_decision() logging.info("grow decision: {}".format(grow_decisions)) hpc_cn_nodes: List[HpcNode] = hpcpack_rest_client.list_computenodes() hpc_cn_nodes = [n for n in hpc_cn_nodes if n.active] # This function will link node history items, cc nodes and hpc nodes node_history.synchronize(cc_nodes, hpc_cn_nodes) cc_nodearrays = set([b.nodearray for b in node_mgr.get_buckets()]) logging.info("Current node arrays in cyclecloud: {}".format(cc_nodearrays)) # Create HPC node groups for CC node arrays cc_map_hpc_groups = ["CycleCloudNodes"] + list(cc_nodearrays) for cc_grp in cc_map_hpc_groups: if ci_notin(cc_grp, hpc_node_groups): logging.info("Create HPC node group: {}".format(cc_grp)) hpcpack_rest_client.add_node_group(cc_grp, "Cycle Cloud Node group") # Add HPC nodes into corresponding node groups add_cc_tag_nodes = [ n.name for n in hpc_cn_nodes if n.shall_addcyclecloudtag ] if len(add_cc_tag_nodes) > 0: logging.info( "Adding HPC nodes to node group CycleCloudNodes: {}".format( add_cc_tag_nodes)) hpcpack_rest_client.add_node_to_node_group("CycleCloudNodes", add_cc_tag_nodes) for cc_grp in list(cc_nodearrays): add_array_tag_nodes = [ n.name for n in hpc_cn_nodes if n.shall_addnodearraytag and ci_equals(n.cc_nodearray, cc_grp) ] if len(add_array_tag_nodes) > 0: logging.info("Adding HPC nodes to node group {}: {}".format( cc_grp, add_array_tag_nodes)) hpcpack_rest_client.add_node_to_node_group(cc_grp, add_array_tag_nodes) # Possible values for HPC NodeState (states marked with * shall not occur for CC nodes): # Unknown, Provisioning, Offline, Starting, Online, Draining, Rejected(*), Removing, NotDeployed(*), Stopping(*) # Remove the following HPC Pack nodes: # 1. The corresponding CC node already removed # 2. The corresponding CC node is stopped and HPC node is not assigned a node template # Take offline the following HPC Pack nodes: # 1. The corresponding CC node is stopped or is going to stop hpc_nodes_to_remove = [ n.name for n in hpc_cn_nodes if n.removed_cc_node or (n.stopped_cc_node and not n.template_assigned) ] hpc_nodes_to_take_offline = [ n.name for n in hpc_cn_nodes if n.stopped_cc_node and ci_equals(n.state, "Online") ] if len(hpc_nodes_to_remove) > 0: logging.info("Removing the HPC nodes: {}".format(hpc_nodes_to_remove)) if dry_run: logging.info("Dry-run: no real action") else: hpcpack_rest_client.remove_nodes(hpc_nodes_to_remove) hpc_cn_nodes = [ n for n in hpc_cn_nodes if not (n.stopped_cc_node or n.removed_cc_node) ] # Assign default node template for unapproved CC node hpc_nodes_to_assign_template = [ n.name for n in hpc_cn_nodes if n.bound_cc_node and not n.template_assigned ] if len(hpc_nodes_to_assign_template) > 0: logging.info( "Assigning default node template for the HPC nodes: {}".format( hpc_nodes_to_assign_template)) if dry_run: logging.info("Dry-run: no real action") else: hpcpack_rest_client.assign_default_compute_node_template( hpc_nodes_to_assign_template) ### Start scale up checking: logging.info("Start scale up checking ...") if ctx_handler: ctx_handler.set_context("[scale-up]") hpc_nodes_with_active_cc = [ n for n in hpc_cn_nodes if n.template_assigned and n.bound_cc_node ] # Exclude the already online healthy HPC nodes before calling node_mgr.allocate for hpc_node in hpc_nodes_with_active_cc: if hpc_node.ready_for_job: hpc_node.bound_cc_node.closed = True # Terminate the provisioning timeout CC nodes cc_node_to_terminate: List[Node] = [] for cc_node in cc_nodes: if ci_equals(cc_node.target_state, 'Deallocated') or ci_equals( cc_node.target_state, 'Terminated') or cc_node.create_time_remaining: continue nhi = node_history.find(cc_id=cc_node.delayed_node_id.node_id) if not nhi.hpc_id: cc_node.closed = True cc_node_to_terminate.append(cc_node) else: hpc_node = ci_find_one(hpc_nodes_with_active_cc, nhi.hpc_id, lambda n: n.id) if hpc_node and hpc_node.error: cc_node.closed = True cc_node_to_terminate.append(cc_node) # "ComputeNodes", "CycleCloudNodes", "AzureIaaSNodes" are all treated as default # grow_by_socket not supported yet, treat as grow_by_node defaultGroups = [ "Default", "ComputeNodes", "AzureIaaSNodes", "CycleCloudNodes" ] default_cores_to_grow = default_nodes_to_grow = 0.0 # If the current CC nodes in the node array cannot satisfy the grow decision, the group is hungry # For a hungry group, no idle check is required if the node health is OK group_hungry: Dict[str, bool] = {} nbrNewNodes: int = 0 grow_groups = list(grow_decisions.keys()) for grp in grow_groups: tmp = grow_decisions.pop(grp) if not (tmp.cores_to_grow + tmp.nodes_to_grow + tmp.sockets_to_grow): continue if ci_in(grp, defaultGroups): default_cores_to_grow += tmp.cores_to_grow default_nodes_to_grow += tmp.nodes_to_grow + tmp.sockets_to_grow continue if ci_notin(grp, cc_nodearrays): logging.warning( "No mapping node array for the grow requirement {}:{}".format( grp, grow_decisions[grp])) grow_decisions.pop(grp) continue group_hungry[grp] = False array = ci_lookup(grp, cc_nodearrays) selector = {'ncpus': 1, 'node.nodearray': [array]} target_cores = math.ceil(tmp.cores_to_grow) target_nodes = math.ceil(tmp.nodes_to_grow + tmp.sockets_to_grow) if target_nodes: logging.info("Allocate: {} Target Nodes: {}".format( selector, target_nodes)) result = node_mgr.allocate(selector, node_count=target_nodes) logging.info(result) if not result or result.total_slots < target_nodes: group_hungry[grp] = True if target_cores: logging.info("Allocate: {} Target Cores: {}".format( selector, target_cores)) result = node_mgr.allocate(selector, slot_count=target_cores) logging.info(result) if not result or result.total_slots < target_cores: group_hungry[grp] = True if len(node_mgr.new_nodes) > nbrNewNodes: group_hungry[grp] = True nbrNewNodes = len(node_mgr.new_nodes) # We then check the grow decision for the default node groups: checkShrinkNeeded = True growForDefaultGroup = True if default_nodes_to_grow or default_cores_to_grow else False if growForDefaultGroup: selector = {'ncpus': 1} if default_nodes_to_grow: target_nodes = math.ceil(default_nodes_to_grow) logging.info("Allocate: {} Target Nodes: {}".format( selector, target_nodes)) result = node_mgr.allocate({'ncpus': 1}, node_count=target_nodes) if not result or result.total_slots < target_nodes: checkShrinkNeeded = False if default_cores_to_grow: target_cores = math.ceil(default_cores_to_grow) logging.info("Allocate: {} Target Cores: {}".format( selector, target_cores)) result = node_mgr.allocate({'ncpus': 1}, slot_count=target_cores) if not result or result.total_slots < target_cores: checkShrinkNeeded = False if len(node_mgr.new_nodes) > nbrNewNodes: checkShrinkNeeded = False nbrNewNodes = len(node_mgr.new_nodes) if nbrNewNodes > 0: logging.info("Need to Allocate {} nodes in total".format(nbrNewNodes)) if dry_run: logging.info("Dry-run: skipping node bootup...") else: logging.info("Allocating {} nodes in total".format( len(node_mgr.new_nodes))) bootup_result: BootupResult = node_mgr.bootup() logging.info(bootup_result) if bootup_result and bootup_result.nodes: for cc_node in bootup_result.nodes: nhi = node_history.find( cc_id=cc_node.delayed_node_id.node_id) if nhi is None: nhi = node_history.insert( NodeHistoryItem(cc_node.delayed_node_id.node_id)) else: nhi.restart() else: logging.info("No need to allocate new nodes ...") ### Start the shrink checking if ctx_handler: ctx_handler.set_context("[scale-down]") cc_node_to_shutdown: List[Node] = [] if not checkShrinkNeeded: logging.info("No shrink check at this round ...") if not dry_run: for nhi in node_history.items: if not nhi.stopped and nhi.hpc_id: nhi.idle_from = None else: logging.info("Start scale down checking ...") # By default, we check idle for active CC nodes in HPC Pack with 'Offline', 'Starting', 'Online', 'Draining' state candidate_idle_check_nodes = [ n for n in hpc_nodes_with_active_cc if (not n.bound_cc_node.keep_alive) and ci_in(n.state, ["Offline", "Starting", "Online", "Draining"]) ] # We can exclude some nodes from idle checking: # 1. If HPC Pack ask for grow in default node group(s), all healthy ONLINE nodes are considered as busy # 2. If HPC Pack ask for grow in certain node group, all healthy ONLINE nodes in that node group are considered as busy # 3. If a node group is hungry (new CC required or grow request not satisfied), no idle check needed for all nodes in that node array if growForDefaultGroup: candidate_idle_check_nodes = [ n for n in candidate_idle_check_nodes if not n.ready_for_job ] for grp, hungry in group_hungry.items(): if hungry: candidate_idle_check_nodes = [ n for n in candidate_idle_check_nodes if not ci_equals(grp, n.cc_nodearray) ] elif not growForDefaultGroup: candidate_idle_check_nodes = [ n for n in candidate_idle_check_nodes if not (ci_equals(grp, n.cc_nodearray) and n.ready_for_job) ] curtime = datetime.utcnow() # Offline node must be idle idle_node_names = [ n.name for n in candidate_idle_check_nodes if ci_equals(n.state, 'Offline') ] if len(candidate_idle_check_nodes) > len(idle_node_names): idle_nodes = hpcpack_rest_client.check_nodes_idle([ n.name for n in candidate_idle_check_nodes if not ci_equals(n.state, 'Offline') ]) if len(idle_nodes) > 0: idle_node_names.extend([n.node_name for n in idle_nodes]) if len(idle_node_names) > 0: logging.info( "The following node is idle: {}".format(idle_node_names)) else: logging.info("No idle node found in this round.") retention_days = autoscale_config.get("vm_retention_days") or 7 for nhi in node_history.items: if nhi.stopped: if nhi.stop_time + timedelta( days=retention_days) < datetime.utcnow(): cc_node = cc_nodes_by_id.get(nhi.cc_id) if cc_node is not None: cc_node_to_terminate.append(cc_node) continue if ci_in(nhi.hostname, idle_node_names): if nhi.idle_from is None: nhi.idle_from = curtime elif nhi.idle_timeout(idle_timeout_seconds): nhi.stop_time = curtime cc_node = cc_nodes_by_id.get(nhi.cc_id) if cc_node is not None: cc_node_to_shutdown.append(cc_node) else: nhi.idle_from = None shrinking_cc_node_ids = [ n.delayed_node_id.node_id for n in cc_node_to_terminate ] shrinking_cc_node_ids.extend( [n.delayed_node_id.node_id for n in cc_node_to_shutdown]) hpc_nodes_to_bring_online = [ n.name for n in hpc_nodes_with_active_cc if ci_equals(n.state, 'Offline') and not n.error and ci_notin(n.cc_node_id, shrinking_cc_node_ids) ] hpc_nodes_to_take_offline.extend([ n.name for n in hpc_nodes_with_active_cc if ci_equals(n.state, 'Online') and ci_in(n.cc_node_id, shrinking_cc_node_ids) ]) if len(hpc_nodes_to_bring_online) > 0: logging.info("Bringing the HPC nodes online: {}".format( hpc_nodes_to_bring_online)) if dry_run: logging.info("Dry-run: no real action") else: hpcpack_rest_client.bring_nodes_online(hpc_nodes_to_bring_online) if len(hpc_nodes_to_take_offline) > 0: logging.info("Taking the HPC nodes offline: {}".format( hpc_nodes_to_take_offline)) if dry_run: logging.info("Dry-run: no real action") else: hpcpack_rest_client.take_nodes_offline(hpc_nodes_to_take_offline) if len(cc_node_to_shutdown) > 0: logging.info("Shut down the following Cycle cloud node: {}".format( [cn.name for cn in cc_node_to_shutdown])) if dry_run: logging.info("Dry-run: skip ...") else: node_mgr.shutdown_nodes(cc_node_to_shutdown) if len(cc_node_to_terminate) > 0: logging.info( "Terminating the following provisioning-timeout Cycle cloud nodes: {}" .format([cn.name for cn in cc_node_to_terminate])) if dry_run: logging.info("Dry-run: skip ...") else: node_mgr.terminate_nodes(cc_node_to_terminate) if not dry_run: logging.info("Save node history: {}".format(node_history)) node_history.save()
def new_rest_client(config: Dict[str, Any]) -> HpcRestClient: hpcpack_config = config.get('hpcpack') or {} hpc_pem_file = hpcpack_config.get('pem') hn_hostname = hpcpack_config.get('hn_hostname') return HpcRestClient(config, pem=hpc_pem_file, hostname=hn_hostname) if __name__ == "__main__": config_file = "" if len(sys.argv) > 1: config_file = sys.argv[1] dry_run = False if len(sys.argv) > 2: dry_run = ci_in(sys.argv[2], ['true', 'dryrun']) ctx_handler = register_result_handler( DefaultContextHandler("[initialization]")) config = load_config(config_file) logging.initialize_logging(config) logging.info( "------------------------------------------------------------------------" ) if config["autoscale"]["start_enabled"]: autoscale_hpcpack(config, ctx_handler=ctx_handler, dry_run=dry_run) else: logging.info("Autoscaler is not enabled")
def _setup_shell_locals(self, config: Dict) -> Dict: """ Provides read only interactive shell. type pbsprohelp() in the shell for more information """ ctx = DefaultContextHandler("[interactive-readonly]") pbs_driver = PBSProDriver(config) pbs_env = self._pbs_env(pbs_driver) def pbsprohelp() -> None: print( "config - dict representing autoscale configuration." ) print( "cli - object representing the CLI commands") print( "pbs_env - object that contains data structures for queues, resources etc" ) print( "queues - dict of queue name -> PBSProQueue object" ) print("jobs - dict of job id -> Autoscale Job") print( "scheduler_nodes - dict of hostname -> node objects. These represent purely what" " the scheduler sees without additional booting nodes / information from CycleCloud" ) print( "resource_definitions - dict of resource name -> PBSProResourceDefinition objects." ) print( "default_scheduler - PBSProScheduler object representing the default scheduler." ) print( "pbs_driver - PBSProDriver object that interacts directly with PBS and implements" " PBS specific behavior for scalelib.") print( "demand_calc - ScaleLib DemandCalculator - pseudo-scheduler that determines the what nodes are unnecessary" ) print( "node_mgr - ScaleLib NodeManager - interacts with CycleCloud for all node related" + " activities - creation, deletion, limits, buckets etc." ) print("pbsprohelp - This help function") # try to make the key "15" instead of "15.hostname" if only # a single submitter was in use num_scheds = len(set([x.name.split(".", 1)[-1] for x in pbs_env.jobs])) if num_scheds == 1: jobs_dict = partition_single(pbs_env.jobs, lambda j: j.name.split(".")[0]) else: jobs_dict = partition_single(pbs_env.jobs, lambda j: j.name) sched_nodes_dict = partition_single(pbs_env.scheduler_nodes, lambda n: n.hostname) pbs_env.queues = clilib.ShellDict(pbs_env.queues) for snode in pbs_env.scheduler_nodes: snode.shellify() pbs_env.resource_definitions = clilib.ShellDict( pbs_env.resource_definitions) demand_calc, _ = self._demand_calc(config, pbs_driver) shell_locals = { "config": config, "cli": self, "ctx": ctx, "pbs_env": pbs_env, "queues": pbs_env.queues, "jobs": clilib.ShellDict(jobs_dict, "j"), "scheduler_nodes": clilib.ShellDict(sched_nodes_dict), "resource_definitions": pbs_env.resource_definitions, "default_scheduler": pbs_env.default_scheduler, "pbs_driver": pbs_driver, "demand_calc": demand_calc, "node_mgr": demand_calc.node_mgr, "pbsprohelp": pbsprohelp, } return shell_locals
def shell(config: Dict) -> None: """ Provides read only interactive shell. type gehelp() in the shell for more information """ ctx = DefaultContextHandler("[interactive-readonly]") ge_env = environment.from_qconf(config) ge_driver = autoscaler.new_driver(config, ge_env) config = ge_driver.preprocess_config(config) demand_calc = autoscaler.new_demand_calculator(config, ge_env, ge_driver, ctx) queues = ge_env.queues def gehelp() -> None: print("config - dict representing autoscale configuration.") print("dbconn - Read-only SQLite conn to node history") print("demand_calc - DemandCalculator") print("ge_driver - GEDriver object.") print("jobs - List[Job] from ge_driver") print("node_mgr - NodeManager") print("logging - HPCLogging module") print("queues - GridEngineQueue objects") shell_locals = { "config": config, "ctx": ctx, "ge_driver": ge_driver, "demand_calc": demand_calc, "node_mgr": demand_calc.node_mgr, "jobs": ge_env.jobs, "dbconn": demand_calc.node_history.conn, "gehelp": gehelp, "queues": queues, "ge_env": ge_env, } banner = "\nCycleCloud GE Autoscale Shell" interpreter = ReraiseAssertionInterpreter(locals=shell_locals) try: __import__("readline") # some magic - create a completer that is bound to the locals in this interpreter and not # the __main__ interpreter. interpreter.push("import readline, rlcompleter") interpreter.push('readline.parse_and_bind("tab: complete")') interpreter.push("_completer = rlcompleter.Completer(locals())") interpreter.push("def _complete_helper(text, state):") interpreter.push(" ret = _completer.complete(text, state)") interpreter.push(' ret = ret + ")" if ret[-1] == "(" else ret') interpreter.push(" return ret") interpreter.push("") interpreter.push("readline.set_completer(_complete_helper)") for item in interpreter.history_lines: try: if '"""' in item: interpreter.push( "readline.add_history('''%s''')" % item.rstrip("\n") ) else: interpreter.push( 'readline.add_history("""%s""")' % item.rstrip("\n") ) except Exception: pass interpreter.push("from hpc.autoscale.job.job import Job\n") interpreter.push("from hpc.autoscale import hpclogging as logging\n") except ImportError: banner += ( "\nWARNING: `readline` is not installed, so autocomplete will not work." ) interpreter.interact(banner=banner)
def setup_function(function: Any) -> None: SchedulerNode.ignore_hostnames = True register_result_handler( DefaultContextHandler("[{}]".format(function.__name__)))