def _late_binding_install(
cls,
cluster_deployment: ClusterDeployment,
agent_cluster_install: AgentClusterInstall,
agents: List["Agent"],
nodes: Nodes,
hold_installation: bool = False,
) -> None:
cls._bind_all(cluster_deployment, agents)
cls._set_agent_cluster_install_machine_cidr(agent_cluster_install, nodes)
if len(nodes) == 1:
cls.set_single_node_ip(cluster_deployment, nodes)
agent_cluster_install.wait_to_be_ready(ready=True)
Agent.wait_for_agents_to_be_bound(agents)
if not hold_installation:
cls._wait_for_install(agent_cluster_install, agents)
def kube_api_test_prepare_late_binding_cluster(
kube_api_context: KubeAPIContext,
cluster_config: ClusterConfig,
num_controlplane_agents: int,
*,
proxy_server=None,
is_ipv4=True,
hold_installation=False,
) -> (ClusterDeployment, AgentClusterInstall, ClusterConfig):
cluster_name = cluster_config.cluster_name.get()
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-agent-cluster-install",
namespace=global_variables.spoke_namespace,
)
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-secret",
namespace=global_variables.spoke_namespace,
)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(
kube_api_client=kube_api_context.api_client,
name=cluster_name,
namespace=global_variables.spoke_namespace,
)
cluster_deployment.create(
agent_cluster_install_ref=agent_cluster_install.ref,
secret=secret,
)
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=deploy_image_set(cluster_name, kube_api_context),
cluster_cidr=cluster_config.cluster_networks[0].cidr,
host_prefix=cluster_config.cluster_networks[0].host_prefix,
service_network=cluster_config.service_networks[0].cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=num_controlplane_agents,
hold_installation=hold_installation,
worker_agents=0,
)
agent_cluster_install.wait_to_be_ready(False)
return cluster_deployment, agent_cluster_install, cluster_config
def download_logs_kube_api(api_client: ApiClient, cluster_name: str,
namespace: str, dest: str, must_gather: bool,
management_kubeconfig: str):
cluster_deployment = ClusterDeployment(
kube_api_client=api_client,
name=cluster_name,
namespace=namespace,
)
agent_cluster_install = AgentClusterInstall(
kube_api_client=api_client,
name=cluster_deployment.get()["spec"]["clusterInstallRef"]["name"],
namespace=namespace,
)
output_folder = os.path.join(dest, f"{cluster_name}")
recreate_folder(output_folder)
try:
with SuppressAndLog(requests.exceptions.RequestException,
ConnectionError):
collect_debug_info_from_cluster(cluster_deployment,
agent_cluster_install,
output_folder)
if must_gather:
recreate_folder(os.path.join(output_folder, "must-gather"))
with SuppressAndLog(Exception):
# in case of hypershift
if namespace.startswith("clusters"):
log.info("Dumping hypershift files")
hypershift = HyperShift(name=cluster_name)
hypershift.dump(os.path.join(output_folder, "dump"),
management_kubeconfig)
else:
_must_gather_kube_api(cluster_name, cluster_deployment,
agent_cluster_install, output_folder)
finally:
run_command(f"chmod -R ugo+rx '{output_folder}'")
def prepare_late_binding_cluster(
self,
kube_api_context: KubeAPIContext,
cluster_config: ClusterConfig,
num_controlplane_agents: int,
*,
hold_installation: bool = False,
) -> (ClusterDeployment, AgentClusterInstall, ClusterConfig):
cluster_name = cluster_config.cluster_name.get()
api_client = kube_api_context.api_client
spoke_namespace = kube_api_context.spoke_namespace
agent_cluster_install = AgentClusterInstall(
api_client, f"{cluster_name}-agent-cluster-install", spoke_namespace
)
secret = Secret(api_client, f"{cluster_name}-secret", spoke_namespace)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(api_client, cluster_name, spoke_namespace)
cluster_deployment.create(agent_cluster_install_ref=agent_cluster_install.ref, secret=secret)
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=self.deploy_image_set(cluster_name, api_client),
cluster_cidr=cluster_config.cluster_networks[0].cidr,
host_prefix=cluster_config.cluster_networks[0].host_prefix,
service_network=cluster_config.service_networks[0].cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=num_controlplane_agents,
hold_installation=hold_installation,
worker_agents=0,
)
agent_cluster_install.wait_to_be_ready(ready=False)
return cluster_deployment, agent_cluster_install, cluster_config
def _set_agent_cluster_install_machine_cidr(cls, agent_cluster_install: AgentClusterInstall, nodes: Nodes):
machine_cidr = nodes.controller.get_primary_machine_cidr()
agent_cluster_install.set_machinenetwork(machine_cidr)
def _wait_for_install(cls, agent_cluster_install: AgentClusterInstall, agents: List[Agent]):
agent_cluster_install.wait_to_be_ready(ready=True)
agent_cluster_install.wait_to_be_installing()
Agent.wait_for_agents_to_install(agents)
agent_cluster_install.wait_to_be_installed()
def kube_api_test(
self,
kube_api_context: KubeAPIContext,
nodes: Nodes,
cluster_config: ClusterConfig,
prepared_controller_configuration: BaseNodeConfig,
infra_env_configuration: BaseInfraEnvConfig,
proxy_server: Optional[Callable] = None,
*,
is_disconnected: bool = False,
):
cluster_name = cluster_config.cluster_name.get()
api_client = kube_api_context.api_client
spoke_namespace = kube_api_context.spoke_namespace
# TODO resolve it from the service if the node controller doesn't have this information
# (please see cluster.get_primary_machine_cidr())
agent_cluster_install = AgentClusterInstall(
api_client, f"{cluster_name}-agent-cluster-install", spoke_namespace
)
secret = Secret(api_client, f"{cluster_name}-secret", spoke_namespace)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(api_client, cluster_name, spoke_namespace)
cluster_deployment.create(agent_cluster_install_ref=agent_cluster_install.ref, secret=secret)
proxy = self.setup_proxy(nodes, cluster_config, proxy_server)
if is_disconnected:
log.info("getting ignition and install config override for disconnected install")
ca_bundle = self.get_ca_bundle_from_hub(spoke_namespace)
self.patch_install_config_with_ca_bundle(cluster_deployment, ca_bundle)
ignition_config_override = self.get_ignition_config_override(ca_bundle)
else:
ignition_config_override = None
infra_env = InfraEnv(api_client, f"{cluster_name}-infra-env", spoke_namespace)
infraenv = infra_env.create(
cluster_deployment, secret, proxy, ignition_config_override, ssh_pub_key=cluster_config.ssh_public_key
)
cluster_config.iso_download_path = utils.get_iso_download_path(infraenv.get("metadata", {}).get("name"))
nodes.prepare_nodes()
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=self.deploy_image_set(cluster_name, api_client),
cluster_cidr=cluster_config.cluster_networks[0].cidr,
host_prefix=cluster_config.cluster_networks[0].host_prefix,
service_network=cluster_config.service_networks[0].cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=nodes.masters_count,
worker_agents=nodes.workers_count,
proxy=proxy.as_dict() if proxy else {},
)
agent_cluster_install.wait_to_be_ready(ready=False)
if infra_env_configuration.is_static_ip:
self.apply_static_network_config(kube_api_context, nodes, cluster_name)
agents = self.start_nodes(nodes, infra_env, cluster_config, infra_env_configuration.is_static_ip)
if len(nodes) == 1:
# for single node set the cidr and take the actual ip from the host
# the vips is the ip of the host
self._set_agent_cluster_install_machine_cidr(agent_cluster_install, nodes)
# wait till the ip is set for the node and read it from its inventory
self.set_single_node_ip(cluster_deployment, nodes)
api_vip = ingress_vip = get_ip_for_single_node(cluster_deployment, nodes.is_ipv4)
else:
# for multi node allocate 2 address at a safe distance from the beginning
# of the available address block to allow enough addresses for workers
access_vips = nodes.controller.get_ingress_and_api_vips()
api_vip = access_vips["api_vip"]
ingress_vip = access_vips["ingress_vip"]
# patch the aci with the vips. The cidr will be derived from the range
agent_cluster_install.set_api_vip(api_vip)
agent_cluster_install.set_ingress_vip(ingress_vip)
nodes.controller.set_dns(api_ip=api_vip, ingress_ip=ingress_vip)
log.info("Waiting for install")
self._wait_for_install(agent_cluster_install, agents)
def kube_api_test(
kube_api_context,
nodes: Nodes,
cluster_config: ClusterConfig,
proxy_server=None,
*,
is_ipv4=True,
is_disconnected=False,
):
cluster_name = cluster_config.cluster_name.get()
# TODO resolve it from the service if the node controller doesn't have this information
# (please see cluster.get_primary_machine_cidr())
machine_cidr = nodes.controller.get_primary_machine_cidr()
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-agent-cluster-install",
namespace=global_variables.spoke_namespace,
)
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-secret",
namespace=global_variables.spoke_namespace,
)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(
kube_api_client=kube_api_context.api_client,
name=cluster_name,
namespace=global_variables.spoke_namespace,
)
cluster_deployment.create(
agent_cluster_install_ref=agent_cluster_install.ref,
secret=secret,
)
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=deploy_image_set(cluster_name, kube_api_context),
cluster_cidr=cluster_config.cluster_networks[0].cidr,
host_prefix=cluster_config.cluster_networks[0].host_prefix,
service_network=cluster_config.service_networks[0].cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=nodes.controller.params.master_count,
worker_agents=nodes.controller.params.worker_count,
machine_cidr=machine_cidr,
)
agent_cluster_install.wait_to_be_ready(False)
if is_disconnected:
log.info("getting igntion and install config override for disconected install")
ca_bundle = get_ca_bundle_from_hub()
patch_install_config_with_ca_bundle(cluster_deployment, ca_bundle)
ignition_config_override = get_ignition_config_override(ca_bundle)
else:
ignition_config_override = None
proxy = setup_proxy(cluster_config, machine_cidr, cluster_name, proxy_server)
infra_env = InfraEnv(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-infra-env",
namespace=global_variables.spoke_namespace,
)
infra_env.create(
cluster_deployment=cluster_deployment,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=proxy,
ssh_pub_key=cluster_config.ssh_public_key,
)
infra_env.status()
download_iso_from_infra_env(infra_env, cluster_config.iso_download_path)
log.info("iso downloaded, starting nodes")
nodes.start_all()
log.info("waiting for host agent")
agents = cluster_deployment.wait_for_agents(len(nodes))
for agent in agents:
agent.approve()
set_agent_hostname(nodes[0], agent, is_ipv4) # Currently only supports single node
if len(nodes) == 1:
set_single_node_ip(cluster_deployment, nodes, is_ipv4)
log.info("Waiting for agent status verification")
Agent.wait_for_agents_to_install(agents)
agent_cluster_install.wait_to_be_ready(True)
log.info("waiting for agent-cluster-install to be in installing state")
agent_cluster_install.wait_to_be_installing()
try:
log.info("installation started, waiting for completion")
agent_cluster_install.wait_to_be_installed()
log.info("installation completed successfully")
except Exception:
log.exception("Failure during kube-api installation flow:")
collect_debug_info_from_cluster(cluster_deployment, agent_cluster_install)
def execute_kube_api_flow():
log.info("Executing kube-api flow")
cluster_name = f'{args.cluster_name or consts.CLUSTER_PREFIX}-{args.namespace}'
utils.recreate_folder(consts.IMAGE_FOLDER, force_recreate=False)
machine_net = MachineNetwork(args.ipv4, args.ipv6, args.vm_network_cidr, args.vm_network_cidr6, args.ns_index)
kube_client = create_kube_api_client()
cluster_deployment = ClusterDeployment(
kube_api_client=kube_client,
name=cluster_name,
namespace=args.namespace
)
set_tf_config(cluster_name)
secret = Secret(
kube_api_client=kube_client,
name=cluster_name,
namespace=args.namespace,
)
secret.apply(pull_secret=args.pull_secret)
imageSet=ClusterImageSet(
kube_api_client=kube_client,
name=f"{cluster_name}-image-set",
namespace=args.namespace
)
releaseImage = utils.get_openshift_release_image()
imageSet.apply(releaseImage=releaseImage)
ipv4 = args.ipv4 and args.ipv4.lower() in MachineNetwork.YES_VALUES
ipv6 = args.ipv6 and args.ipv6.lower() in MachineNetwork.YES_VALUES
api_vip, ingress_vip = "", ""
if args.master_count > 1:
api_vip, ingress_vip = _get_vips_ips(machine_net)
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_client,
name=f'{cluster_name}-agent-cluster-install',
namespace=args.namespace
)
image_set_ref = ClusterImageSetReference(name=f'{cluster_name}-image-set')
cluster_deployment.apply(
secret=secret,
base_domain=args.base_dns_domain,
agent_cluster_install_ref=agent_cluster_install.ref,
)
agent_cluster_install.apply(
cluster_deployment_ref=cluster_deployment.ref,
api_vip=api_vip,
ingress_vip=ingress_vip,
image_set_ref=image_set_ref,
cluster_cidr=args.cluster_network if ipv4 else args.cluster_network6,
host_prefix=args.host_prefix if ipv4 else args.host_prefix6,
service_network=args.service_network if ipv4 else args.service_network6,
ssh_pub_key=args.ssh_key,
control_plane_agents=args.master_count,
worker_agents=args.number_of_workers,
machine_cidr=get_machine_cidr_from_machine_net(machine_net),
hyperthreading=args.hyperthreading,
)
agent_cluster_install.wait_to_be_ready(False)
apply_static_network_config(
cluster_name=cluster_name,
kube_client=kube_client,
)
image_path = os.path.join(
consts.IMAGE_FOLDER,
f'{args.namespace}-installer-image.iso'
)
log.info("Creating infraEnv")
http_proxy, https_proxy, no_proxy = _get_http_proxy_params(ipv4=ipv4, ipv6=ipv6)
infra_env = InfraEnv(
kube_api_client=kube_client,
name=f"{cluster_name}-infra-env",
namespace=args.namespace
)
infra_env.apply(
cluster_deployment=cluster_deployment,
secret=secret,
proxy=Proxy(
http_proxy=http_proxy,
https_proxy=https_proxy,
no_proxy=no_proxy
),
ssh_pub_key=args.ssh_key,
nmstate_label=cluster_name,
)
infra_env.status()
image_url = infra_env.get_iso_download_url()
utils.download_iso(image_url, image_path)
try:
nodes_flow_kube_api(cluster_name, machine_net, cluster_deployment, agent_cluster_install)
finally:
if not image_path or args.keep_iso:
return
log.info('deleting iso: %s', image_path)
os.unlink(image_path)