def kube_api_test_prepare_late_binding_infraenv(
self, kube_api_context: KubeAPIContext, nodes: Nodes, infraenv_config: InfraEnvConfig
):
api_client = kube_api_context.api_client
spoke_namespace = kube_api_context.spoke_namespace
infraenv_name = infraenv_config.entity_name.get()
infraenv_config.iso_download_path = utils.get_iso_download_path(infraenv_name)
nodes.prepare_nodes()
spoke_namespace = spoke_namespace
secret = Secret(api_client, f"{infraenv_name}-secret", spoke_namespace)
secret.create(pull_secret=infraenv_config.pull_secret)
ignition_config_override = None
infra_env = InfraEnv(api_client, f"{infraenv_name}-infra-env", spoke_namespace)
infra_env.create(
cluster_deployment=None,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=None,
ssh_pub_key=infraenv_config.ssh_public_key,
)
agents = self.start_nodes(nodes, infra_env, infraenv_config, infraenv_config.is_static_ip)
log.info("Waiting for agent status verification")
Agent.wait_for_agents_to_be_ready_for_install(agents)
return infra_env
def _set_host_name_from_node(cls, nodes: List[Node], agent: Agent, is_ipv4: bool) -> None:
"""
Use the MAC address that is listed in the virt node object to find the interface entry
in the host's inventory and take the host name from there
The setting of the hostname is required for IPv6 only, because the nodes are booted with
hostname equal to localhost which is neither unique not legal name for AI host
"""
if is_ipv4:
return
def find_matching_node_and_return_name():
inventory = agent.status().get("inventory", {})
for node in nodes:
for mac_address in node.macs:
for interface in inventory.get("interfaces", []):
if interface["macAddress"].lower() == mac_address.lower():
return node.name
hostname = waiting.wait(
find_matching_node_and_return_name,
timeout_seconds=60,
sleep_seconds=1,
waiting_for=f"agent={agent.ref} to find a hostname",
)
log.info(f"patching agent {agent.ref} with hostname {hostname}")
agent.patch(hostname=hostname)
def scale_down_nodepool_and_wait_for_unbounded_agent(
cls, cluster_deployment: ClusterDeployment, hypershift: HyperShift, spoke_namespace: str, node_count: int
):
agents = cluster_deployment.list_agents()
log.info("Setting node count to %s", node_count)
hypershift.set_nodepool_replicas(node_count)
log.info("waiting for capi provider to remove clusterDeployment ref from the agent")
updated_agents = cluster_deployment.wait_for_agents(node_count, agents_namespace=spoke_namespace)
removed_agent = set(agents) - set(updated_agents)
log.info("Agent: {} removed")
log.info("Waiting for agent to unbind")
Agent.wait_for_agents_to_unbound(list(removed_agent))
log.info("Scale down completed")
def test_kube_api_late_unbinding_ipv4_single_node(
self, unbound_single_node_cluster_hold_installation, unbound_single_node_infraenv
):
infra_env, nodes = unbound_single_node_infraenv
cluster_deployment, agent_cluster_install, cluster_config = unbound_single_node_cluster_hold_installation
agents = infra_env.wait_for_agents(len(nodes))
self._late_binding_install(
cluster_deployment, agent_cluster_install, agents, nodes, is_ipv4=True, hold_installation=True
)
cluster_deployment.delete()
Agent.wait_for_agents_to_unbound(agents)
def set_node_count_and_wait_for_ready_nodes(
cls, cluster_deployment: ClusterDeployment, hypershift: HyperShift, spoke_namespace: str, node_count: int
):
log.info("Setting node count to %s", node_count)
hypershift.set_nodepool_replicas(node_count)
log.info("waiting for capi provider to set clusterDeployment ref on the agent")
agents = cluster_deployment.wait_for_agents(node_count, agents_namespace=spoke_namespace)
log.info("Waiting for agents status verification")
Agent.wait_for_agents_to_install(agents)
log.info("Waiting for node to join the cluster")
hypershift.wait_for_nodes(node_count)
log.info("Waiting for node to become ready")
hypershift.wait_for_nodes(node_count, ready=True)
def test_late_binding_kube_api_sno(self, unbound_single_node_cluster, unbound_single_node_infraenv):
infra_env, nodes = unbound_single_node_infraenv
cluster_deployment, agent_cluster_install, cluster_config = unbound_single_node_cluster
agents = infra_env.wait_for_agents(len(nodes))
assert len(agents) == 1, f"Expected only one agent, found {len(agents)}"
self._late_binding_install(
cluster_deployment, agent_cluster_install, agents, nodes, global_variables.hold_installation
)
if global_variables.hold_installation:
cluster_deployment.delete()
Agent.wait_for_agents_to_unbound(agents)
def set_node_count_and_wait_for_ready_nodes(
cluster_deployment: ClusterDeployment, hypershift: HyperShift, kube_api_context, node_count: int
):
log.info("Setting node count to %s", node_count)
hypershift.set_nodepool_node_count(kube_api_context.api_client, node_count)
log.info("waiting for capi provider to set clusterDeployment ref on the agent")
agents = cluster_deployment.wait_for_agents(node_count, agents_namespace=global_variables.spoke_namespace)
log.info("Waiting for agents status verification")
Agent.wait_for_agents_to_install(agents)
hypershift.download_kubeconfig(kube_api_context.api_client)
log.info("Waiting for node to join the cluster")
hypershift.wait_for_nodes(node_count)
log.info("Waiting for node to become ready")
hypershift.wait_for_nodes(node_count, ready=True)
def kube_api_test_prepare_late_binding_infraenv(
kube_api_context,
nodes: Nodes,
infraenv_config: InfraEnvConfig,
*,
is_ipv4=True):
infraenv_name = infraenv_config.entity_name.get()
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f"{infraenv_name}-secret",
namespace=global_variables.spoke_namespace,
)
secret.create(pull_secret=infraenv_config.pull_secret)
ignition_config_override = None
infra_env = InfraEnv(
kube_api_client=kube_api_context.api_client,
name=f"{infraenv_name}-infra-env",
namespace=global_variables.spoke_namespace,
)
infra_env.create(
cluster_deployment=None,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=None,
ssh_pub_key=infraenv_config.ssh_public_key,
)
infra_env.status()
download_iso_from_infra_env(infra_env, infraenv_config.iso_download_path)
log.info("iso downloaded, starting nodes")
nodes.start_all()
log.info("waiting for host agent")
agents = infra_env.wait_for_agents(len(nodes))
for agent in agents:
agent.approve()
set_agent_hostname(nodes[0], agent,
is_ipv4) # Currently only supports single node
log.info("Waiting for agent status verification")
Agent.wait_for_agents_to_be_ready_for_install(agents)
return infra_env
def _late_binding_install(
cls,
cluster_deployment: ClusterDeployment,
agent_cluster_install: AgentClusterInstall,
agents: List["Agent"],
nodes: Nodes,
hold_installation: bool = False,
) -> None:
cls._bind_all(cluster_deployment, agents)
cls._set_agent_cluster_install_machine_cidr(agent_cluster_install, nodes)
if len(nodes) == 1:
cls.set_single_node_ip(cluster_deployment, nodes)
agent_cluster_install.wait_to_be_ready(ready=True)
Agent.wait_for_agents_to_be_bound(agents)
if not hold_installation:
cls._wait_for_install(agent_cluster_install, agents)
def run_installation_flow_kube_api(
cluster_deployment,
agent_cluster_install,
nodes_number,
kubeconfig_path,
):
log.info("Approving agents")
agents = cluster_deployment.wait_for_agents(nodes_number)
for agent in agents:
agent.approve()
log.info("Waiting for agent status verification")
Agent.wait_for_agents_to_install(agents)
log.info("Waiting for installation to start")
agent_cluster_install.wait_to_be_installing()
log.info("Waiting until cluster finishes installation")
agent_cluster_install.wait_to_be_installed()
log.info("Download kubeconfig-noingress")
agent_cluster_install.download_kubeconfig(kubeconfig_path=kubeconfig_path)
def set_agent_hostname(cls, node: Node, agent: Agent, is_ipv4: bool) -> None:
if is_ipv4:
return
log.info("patching agent hostname=%s", node)
agent.patch(hostname=node.name)
def _wait_for_install(cls, agent_cluster_install: AgentClusterInstall, agents: List[Agent]):
agent_cluster_install.wait_to_be_ready(ready=True)
agent_cluster_install.wait_to_be_installing()
Agent.wait_for_agents_to_install(agents)
agent_cluster_install.wait_to_be_installed()
def kube_api_test(
kube_api_context,
nodes: Nodes,
cluster_config: ClusterConfig,
proxy_server=None,
*,
is_ipv4=True,
is_disconnected=False,
):
cluster_name = cluster_config.cluster_name.get()
# TODO resolve it from the service if the node controller doesn't have this information
# (please see cluster.get_primary_machine_cidr())
machine_cidr = nodes.controller.get_primary_machine_cidr()
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-agent-cluster-install",
namespace=global_variables.spoke_namespace,
)
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-secret",
namespace=global_variables.spoke_namespace,
)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(
kube_api_client=kube_api_context.api_client,
name=cluster_name,
namespace=global_variables.spoke_namespace,
)
cluster_deployment.create(
agent_cluster_install_ref=agent_cluster_install.ref,
secret=secret,
)
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=deploy_image_set(cluster_name, kube_api_context),
cluster_cidr=cluster_config.cluster_networks[0].cidr,
host_prefix=cluster_config.cluster_networks[0].host_prefix,
service_network=cluster_config.service_networks[0].cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=nodes.controller.params.master_count,
worker_agents=nodes.controller.params.worker_count,
machine_cidr=machine_cidr,
)
agent_cluster_install.wait_to_be_ready(False)
if is_disconnected:
log.info("getting igntion and install config override for disconected install")
ca_bundle = get_ca_bundle_from_hub()
patch_install_config_with_ca_bundle(cluster_deployment, ca_bundle)
ignition_config_override = get_ignition_config_override(ca_bundle)
else:
ignition_config_override = None
proxy = setup_proxy(cluster_config, machine_cidr, cluster_name, proxy_server)
infra_env = InfraEnv(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-infra-env",
namespace=global_variables.spoke_namespace,
)
infra_env.create(
cluster_deployment=cluster_deployment,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=proxy,
ssh_pub_key=cluster_config.ssh_public_key,
)
infra_env.status()
download_iso_from_infra_env(infra_env, cluster_config.iso_download_path)
log.info("iso downloaded, starting nodes")
nodes.start_all()
log.info("waiting for host agent")
agents = cluster_deployment.wait_for_agents(len(nodes))
for agent in agents:
agent.approve()
set_agent_hostname(nodes[0], agent, is_ipv4) # Currently only supports single node
if len(nodes) == 1:
set_single_node_ip(cluster_deployment, nodes, is_ipv4)
log.info("Waiting for agent status verification")
Agent.wait_for_agents_to_install(agents)
agent_cluster_install.wait_to_be_ready(True)
log.info("waiting for agent-cluster-install to be in installing state")
agent_cluster_install.wait_to_be_installing()
try:
log.info("installation started, waiting for completion")
agent_cluster_install.wait_to_be_installed()
log.info("installation completed successfully")
except Exception:
log.exception("Failure during kube-api installation flow:")
collect_debug_info_from_cluster(cluster_deployment, agent_cluster_install)
def _wait_for_install(agent_cluster_install, agents):
agent_cluster_install.wait_to_be_ready(True)
agent_cluster_install.wait_to_be_installing()
Agent.wait_for_agents_to_install(agents)
agent_cluster_install.wait_to_be_installed()