def kube_api_test_prepare_late_binding_infraenv(
kube_api_context, nodes: Nodes, infraenv_config: InfraEnvConfig, *, is_ipv4=True
):
infraenv_name = infraenv_config.entity_name.get()
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f"{infraenv_name}-secret",
namespace=global_variables.spoke_namespace,
)
secret.create(pull_secret=infraenv_config.pull_secret)
ignition_config_override = None
infra_env = InfraEnv(
kube_api_client=kube_api_context.api_client,
name=f"{infraenv_name}-infra-env",
namespace=global_variables.spoke_namespace,
)
infra_env.create(
cluster_deployment=None,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=None,
ssh_pub_key=infraenv_config.ssh_public_key,
)
infra_env.status()
download_iso_from_infra_env(infra_env, infraenv_config.iso_download_path)
logger.info("iso downloaded, starting nodes")
nodes.start_all()
logger.info("waiting for host agent")
agents = infra_env.wait_for_agents(len(nodes))
for agent in agents:
agent.approve()
set_agent_hostname(nodes[0], agent, is_ipv4) # Currently only supports single node
logger.info("Waiting for agent status verification")
Agent.wait_for_agents_to_be_ready_for_install(agents)
return infra_env
def kube_api_test_prepare_late_binding_cluster(
kube_api_context, cluster_config: ClusterConfig, num_controlplane_agents, *, proxy_server=None, is_ipv4=True
):
cluster_name = cluster_config.cluster_name.get()
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-agent-cluster-install",
namespace=global_variables.spoke_namespace,
)
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-secret",
namespace=global_variables.spoke_namespace,
)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(
kube_api_client=kube_api_context.api_client,
name=cluster_name,
namespace=global_variables.spoke_namespace,
)
cluster_deployment.create(
agent_cluster_install_ref=agent_cluster_install.ref,
secret=secret,
)
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=deploy_image_set(cluster_name, kube_api_context),
cluster_cidr=cluster_config.cluster_networks[0].cidr,
host_prefix=cluster_config.cluster_networks[0].host_prefix,
service_network=cluster_config.service_networks[0].cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=num_controlplane_agents,
worker_agents=0,
)
agent_cluster_install.wait_to_be_ready(False)
return cluster_deployment, agent_cluster_install, cluster_config
def delete_kube_api_resources_for_namespace(
kube_api_client,
name,
namespace,
*,
secret_name=None,
infraenv_name=None,
nmstate_name=None,
image_set_name=None,
):
CoreV1Api.delete_namespaced_secret = suppress_not_found_error(
fn=CoreV1Api.delete_namespaced_secret,
)
CustomObjectsApi.delete_namespaced_custom_object = suppress_not_found_error(
fn=CustomObjectsApi.delete_namespaced_custom_object
)
cluster_deployment = ClusterDeployment(
kube_api_client=kube_api_client,
name=name,
namespace=namespace,
)
for agent in cluster_deployment.list_agents():
agent.delete()
cluster_deployment.delete()
Secret(
kube_api_client=kube_api_client,
name=secret_name or name,
namespace=namespace,
).delete()
InfraEnv(
kube_api_client=kube_api_client,
name=infraenv_name or f"{name}-infra-env",
namespace=namespace,
).delete()
NMStateConfig(
kube_api_client=kube_api_client,
name=nmstate_name or f"{name}-nmstate-config",
namespace=namespace,
).delete()
ClusterImageSet(
kube_api_client=kube_api_client, name=image_set_name or f"{name}-image-set", namespace=namespace
).delete()
def execute_kube_api_flow():
log.info("Executing kube-api flow")
cluster_name = f'{args.cluster_name or consts.CLUSTER_PREFIX}-{args.namespace}'
utils.recreate_folder(consts.IMAGE_FOLDER, force_recreate=False)
machine_net = MachineNetwork(args.ipv4, args.ipv6, args.vm_network_cidr,
args.vm_network_cidr6, args.ns_index)
kube_client = create_kube_api_client()
cluster_deployment = ClusterDeployment(kube_api_client=kube_client,
name=cluster_name,
namespace=args.namespace)
set_tf_config(cluster_name)
secret = Secret(
kube_api_client=kube_client,
name=cluster_name,
namespace=args.namespace,
)
secret.apply(pull_secret=args.pull_secret)
ipv4 = args.ipv4 and args.ipv4.lower() in MachineNetwork.YES_VALUES
ipv6 = args.ipv6 and args.ipv6.lower() in MachineNetwork.YES_VALUES
api_vip, ingress_vip = "", ""
if args.master_count > 1:
api_vip, ingress_vip = _get_vips_ips(machine_net)
cluster_deployment.apply(
platform=Platform(
api_vip=api_vip,
ingress_vip=ingress_vip,
),
install_strategy=InstallStrategy(
host_prefix=args.host_prefix if ipv4 else args.host_prefix6,
machine_cidr=get_machine_cidr_from_machine_net(machine_net),
cluster_cidr=args.cluster_network
if ipv4 else args.cluster_network6,
service_cidr=args.service_network
if ipv4 else args.service_network6,
ssh_public_key=args.ssh_key,
control_plane_agents=args.master_count,
worker_agents=args.number_of_workers,
),
secret=secret,
base_domain=args.base_dns_domain,
)
cluster_deployment.wait_for_state(consts.ClusterStatus.INSUFFICIENT)
apply_static_network_config(
cluster_name=cluster_name,
kube_client=kube_client,
)
image_path = os.path.join(consts.IMAGE_FOLDER,
f'{args.namespace}-installer-image.iso')
log.info("Creating installenv")
http_proxy, https_proxy, no_proxy = _get_http_proxy_params(ipv4=ipv4,
ipv6=ipv6)
install_env = InfraEnv(kube_api_client=kube_client,
name=f"{cluster_name}-install-env",
namespace=args.namespace)
install_env.apply(cluster_deployment=cluster_deployment,
secret=secret,
proxy=Proxy(http_proxy=http_proxy,
https_proxy=https_proxy,
no_proxy=no_proxy))
install_env.status()
image_url = install_env.get_iso_download_url()
utils.download_iso(image_url, image_path)
try:
nodes_flow_kube_api(cluster_name, machine_net, cluster_deployment)
finally:
if not image_path or args.keep_iso:
return
log.info('deleting iso: %s', image_path)
os.unlink(image_path)
def execute_kube_api_flow():
log.info("Executing kube-api flow")
cluster_name = f'{args.cluster_name or consts.CLUSTER_PREFIX}-{args.namespace}'
utils.recreate_folder(consts.IMAGE_FOLDER, force_recreate=False)
machine_net = MachineNetwork(args.ipv4, args.ipv6, args.vm_network_cidr, args.vm_network_cidr6, args.ns_index)
kube_client = create_kube_api_client()
cluster_deployment = ClusterDeployment(
kube_api_client=kube_client,
name=cluster_name,
namespace=args.namespace
)
set_tf_config(cluster_name)
secret = Secret(
kube_api_client=kube_client,
name=cluster_name,
namespace=args.namespace,
)
secret.apply(pull_secret=args.pull_secret)
imageSet=ClusterImageSet(
kube_api_client=kube_client,
name=f"{cluster_name}-image-set",
namespace=args.namespace
)
releaseImage=utils.get_env('OPENSHIFT_INSTALL_RELEASE_IMAGE', utils.get_openshift_release_image("4.8"))
imageSet.apply(releaseImage=releaseImage)
ipv4 = args.ipv4 and args.ipv4.lower() in MachineNetwork.YES_VALUES
ipv6 = args.ipv6 and args.ipv6.lower() in MachineNetwork.YES_VALUES
api_vip, ingress_vip = "", ""
if args.master_count > 1:
api_vip, ingress_vip = _get_vips_ips(machine_net)
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_client,
name=f'{cluster_name}-agent-cluster-install',
namespace=args.namespace
)
image_set_ref = ClusterImageSetReference(name=f'{cluster_name}-image-set')
cluster_deployment.apply(
secret=secret,
base_domain=args.base_dns_domain,
agent_cluster_install_ref=agent_cluster_install.ref,
)
agent_cluster_install.apply(
cluster_deployment_ref=cluster_deployment.ref,
api_vip=api_vip,
ingress_vip=ingress_vip,
image_set_ref=image_set_ref,
cluster_cidr=args.cluster_network if ipv4 else args.cluster_network6,
host_prefix=args.host_prefix if ipv4 else args.host_prefix6,
service_network=args.service_network if ipv4 else args.service_network6,
ssh_pub_key=args.ssh_key,
control_plane_agents=args.master_count,
worker_agents=args.number_of_workers,
machine_cidr=get_machine_cidr_from_machine_net(machine_net),
)
agent_cluster_install.wait_to_be_ready(False)
apply_static_network_config(
cluster_name=cluster_name,
kube_client=kube_client,
)
image_path = os.path.join(
consts.IMAGE_FOLDER,
f'{args.namespace}-installer-image.iso'
)
log.info("Creating infraEnv")
http_proxy, https_proxy, no_proxy = _get_http_proxy_params(ipv4=ipv4, ipv6=ipv6)
infra_env = InfraEnv(
kube_api_client=kube_client,
name=f"{cluster_name}-infra-env",
namespace=args.namespace
)
infra_env.apply(
cluster_deployment=cluster_deployment,
secret=secret,
proxy=Proxy(
http_proxy=http_proxy,
https_proxy=https_proxy,
no_proxy=no_proxy
),
ssh_pub_key=args.ssh_key,
nmstate_label=cluster_name,
)
infra_env.status()
image_url = infra_env.get_iso_download_url()
utils.download_iso(image_url, image_path)
try:
nodes_flow_kube_api(cluster_name, machine_net, cluster_deployment, agent_cluster_install)
finally:
if not image_path or args.keep_iso:
return
log.info('deleting iso: %s', image_path)
os.unlink(image_path)
def kube_api_test(kube_api_context,
nodes,
cluster_config,
proxy_server=None,
*,
is_ipv4=True,
is_disconnected=False):
cluster_name = nodes.controller.cluster_name
machine_cidr = nodes.controller.get_machine_cidr()
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_api_context.api_client,
name=f'{cluster_name}-agent-cluster-install',
)
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f'{cluster_name}-secret',
)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(
kube_api_client=kube_api_context.api_client,
name=cluster_name,
)
cluster_deployment.create(
agent_cluster_install_ref=agent_cluster_install.ref,
secret=secret,
)
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=deploy_image_set(cluster_name, kube_api_context),
cluster_cidr=cluster_config.cluster_network_cidr,
host_prefix=cluster_config.cluster_network_host_prefix,
service_network=cluster_config.service_network_cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=nodes.controller.params.master_count,
worker_agents=nodes.controller.params.worker_count,
machine_cidr=machine_cidr,
)
agent_cluster_install.wait_to_be_ready(False)
if is_disconnected:
logger.info(
'getting igntion and install config override for disconected install'
)
ca_bundle = get_ca_bundle_from_hub()
patch_install_config_with_ca_bundle(cluster_deployment, ca_bundle)
ignition_config_override = get_ignition_config_override(ca_bundle)
else:
ignition_config_override = None
proxy = setup_proxy(cluster_config, machine_cidr, cluster_name,
proxy_server)
infra_env = InfraEnv(
kube_api_client=kube_api_context.api_client,
name=f'{cluster_name}-infra-env',
)
infra_env.create(
cluster_deployment=cluster_deployment,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=proxy,
ssh_pub_key=cluster_config.ssh_public_key,
)
infra_env.status()
download_iso_from_infra_env(infra_env, cluster_config)
logger.info('iso downloaded, starting nodes')
nodes.start_all()
logger.info('waiting for host agent')
for agent in cluster_deployment.wait_for_agents(len(nodes)):
agent.approve()
set_agent_hostname(nodes[0], agent,
is_ipv4) # Currently only supports single node
if len(nodes) == 1:
set_single_node_ip(cluster_deployment, nodes, is_ipv4)
agent_cluster_install.wait_to_be_ready(True)
logger.info('waiting for agent-cluster-install to be in installing state')
agent_cluster_install.wait_to_be_installing()
try:
logger.info('installation started, waiting for completion')
agent_cluster_install.wait_to_be_installed()
logger.info('installation completed successfully')
except Exception:
logger.exception(f"Failure during kube-api installation flow:")
collect_debug_info_from_cluster(cluster_deployment,
agent_cluster_install)
def kube_api_test(
kube_api_context,
nodes: Nodes,
cluster_config: ClusterConfig,
proxy_server=None,
*,
is_ipv4=True,
is_disconnected=False,
):
cluster_name = cluster_config.cluster_name.get()
# TODO resolve it from the service if the node controller doesn't have this information
# (please see cluster.get_primary_machine_cidr())
machine_cidr = nodes.controller.get_primary_machine_cidr()
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-agent-cluster-install",
namespace=global_variables.spoke_namespace,
)
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-secret",
namespace=global_variables.spoke_namespace,
)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(
kube_api_client=kube_api_context.api_client,
name=cluster_name,
namespace=global_variables.spoke_namespace,
)
cluster_deployment.create(
agent_cluster_install_ref=agent_cluster_install.ref,
secret=secret,
)
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=deploy_image_set(cluster_name, kube_api_context),
cluster_cidr=cluster_config.cluster_networks[0].cidr,
host_prefix=cluster_config.cluster_networks[0].host_prefix,
service_network=cluster_config.service_networks[0].cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=nodes.controller.params.master_count,
worker_agents=nodes.controller.params.worker_count,
machine_cidr=machine_cidr,
)
agent_cluster_install.wait_to_be_ready(False)
if is_disconnected:
logger.info("getting igntion and install config override for disconected install")
ca_bundle = get_ca_bundle_from_hub()
patch_install_config_with_ca_bundle(cluster_deployment, ca_bundle)
ignition_config_override = get_ignition_config_override(ca_bundle)
else:
ignition_config_override = None
proxy = setup_proxy(cluster_config, machine_cidr, cluster_name, proxy_server)
infra_env = InfraEnv(
kube_api_client=kube_api_context.api_client,
name=f"{cluster_name}-infra-env",
namespace=global_variables.spoke_namespace,
)
infra_env.create(
cluster_deployment=cluster_deployment,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=proxy,
ssh_pub_key=cluster_config.ssh_public_key,
)
infra_env.status()
download_iso_from_infra_env(infra_env, cluster_config.iso_download_path)
logger.info("iso downloaded, starting nodes")
nodes.start_all()
logger.info("waiting for host agent")
agents = cluster_deployment.wait_for_agents(len(nodes))
for agent in agents:
agent.approve()
set_agent_hostname(nodes[0], agent, is_ipv4) # Currently only supports single node
if len(nodes) == 1:
set_single_node_ip(cluster_deployment, nodes, is_ipv4)
logger.info("Waiting for agent status verification")
Agent.wait_for_agents_to_install(agents)
agent_cluster_install.wait_to_be_ready(True)
logger.info("waiting for agent-cluster-install to be in installing state")
agent_cluster_install.wait_to_be_installing()
try:
logger.info("installation started, waiting for completion")
agent_cluster_install.wait_to_be_installed()
logger.info("installation completed successfully")
except Exception:
logger.exception("Failure during kube-api installation flow:")
collect_debug_info_from_cluster(cluster_deployment, agent_cluster_install)
def kube_api_test(kube_api_context,
nodes,
proxy_server=None,
*,
is_ipv4=True,
is_disconnected=False):
cluster_name = nodes.controller.cluster_name
machine_cidr = nodes.controller.get_machine_cidr()
agent_cluster_install = AgentClusterInstall(
kube_api_client=kube_api_context.api_client,
name=f'{cluster_name}-agent-cluster-install',
)
secret = Secret(
kube_api_client=kube_api_context.api_client,
name=f'{cluster_name}-secret',
)
secret.create(pull_secret=env_variables['pull_secret'])
cluster_deployment = ClusterDeployment(
kube_api_client=kube_api_context.api_client,
name=cluster_name,
)
cluster_deployment.create(
agent_cluster_install_ref=agent_cluster_install.ref,
secret=secret,
)
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=deploy_image_set(cluster_name, kube_api_context),
cluster_cidr=env_variables['cluster_cidr'],
host_prefix=env_variables['host_prefix'],
service_network=env_variables['service_cidr'],
ssh_pub_key=env_variables['ssh_public_key'],
control_plane_agents=nodes.controller.params.master_count,
worker_agents=nodes.controller.params.worker_count,
machine_cidr=machine_cidr,
)
agent_cluster_install.wait_to_be_ready(False)
if is_disconnected:
logger.info(
'getting igntion and install config override for disconected install'
)
ca_bundle = get_ca_bundle_from_hub()
patch_install_config_with_ca_bundle(cluster_deployment, ca_bundle)
ignition_config_override = get_ignition_config_override(ca_bundle)
else:
ignition_config_override = None
proxy = setup_proxy(machine_cidr, cluster_name, proxy_server)
infra_env = InfraEnv(
kube_api_client=kube_api_context.api_client,
name=f'{cluster_name}-infra-env',
)
infra_env.create(
cluster_deployment=cluster_deployment,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=proxy,
ssh_pub_key=env_variables['ssh_public_key'],
)
infra_env.status()
download_iso_from_infra_env(infra_env)
logger.info('iso downloaded, starting nodes')
nodes.start_all()
logger.info('waiting for host agent')
for agent in cluster_deployment.wait_for_agents(len(nodes)):
agent.approve()
if len(hosts) == 1:
set_single_node_ip(cluster_deployment, nodes, is_ipv4)
for node in nodes:
set_agent_hostname(node, agent, is_ipv4)
agent_cluster_install.wait_to_be_ready(True)
logger.info('waiting for agent-cluster-install to be in installing state')
agent_cluster_install.wait_to_be_installing()
logger.info('installation started, waiting for completion')
agent_cluster_install.wait_to_be_installed()
logger.info('installation completed successfully')
def execute_day1_flow(cluster_name):
client = None
cluster = {}
if args.managed_dns_domains:
args.base_dns_domain = args.managed_dns_domains.split(":")[0]
if not args.vm_network_cidr:
net_cidr = IPNetwork('192.168.126.0/24')
net_cidr += args.ns_index
args.vm_network_cidr = str(net_cidr)
if not args.vm_network_cidr6:
net_cidr = IPNetwork('1001:db8::/120')
net_cidr += args.ns_index
args.vm_network_cidr6 = str(net_cidr)
if not args.network_bridge:
args.network_bridge = f'tt{args.ns_index}'
set_tf_config(cluster_name)
image_path = None
image_url = None
image_type = args.iso_image_type
kube_client = None
cluster_deployment = None
machine_net = MachineNetwork(args.ipv4, args.ipv6, args.vm_network_cidr,
args.vm_network_cidr6, args.ns_index)
if not args.image:
utils.recreate_folder(consts.IMAGE_FOLDER, force_recreate=False)
client = assisted_service_api.create_client(
url=utils.get_assisted_service_url_by_args(args=args))
if args.cluster_id:
cluster = client.cluster_get(cluster_id=args.cluster_id)
elif args.kube_api:
kube_client = create_kube_api_client(
str(pathlib.Path("~/.kube/config").expanduser()))
cluster_deployment = ClusterDeployment(kube_api_client=kube_client,
name=cluster_name,
namespace=args.namespace)
secret = Secret(
kube_api_client=kube_client,
name=cluster_name,
namespace=args.namespace,
)
with contextlib.suppress(ApiException):
secret.delete()
secret.create(pull_secret=args.pull_secret)
ipv4 = args.ipv4 and args.ipv4.lower() in MachineNetwork.YES_VALUES
ipv6 = args.ipv6 and args.ipv6.lower() in MachineNetwork.YES_VALUES
api_vip, ingress_vip = "", ""
with contextlib.suppress(ApiException):
cluster_deployment.delete()
cluster_deployment.create(
platform=Platform(
api_vip=api_vip,
ingress_vip=ingress_vip,
),
install_strategy=InstallStrategy(
host_prefix=args.host_prefix
if ipv4 else args.host_prefix6,
machine_cidr=machine_net.machine_cidr_addresses[0],
cluster_cidr=args.cluster_network
if ipv4 else args.cluster_network6,
service_cidr=args.service_network
if ipv4 else args.service_network6,
ssh_public_key=args.ssh_key,
control_plane_agents=args.master_count,
worker_agents=args.number_of_workers,
),
secret=secret,
base_domain=args.base_dns_domain,
)
cluster_deployment.wait_for_state("insufficient")
http_proxy, https_proxy, no_proxy = _get_http_proxy_params(
ipv4=ipv4, ipv6=ipv6)
install_env = InstallEnv(kube_api_client=kube_client,
name=f"{cluster_name}-install-env",
namespace=args.namespace)
with contextlib.suppress(ApiException):
install_env.delete()
install_env.create(cluster_deployment=cluster_deployment,
secret=secret,
proxy=Proxy(http_proxy=http_proxy,
https_proxy=https_proxy,
no_proxy=no_proxy))
install_env.status()
image_url = install_env.get_iso_download_url()
cluster = client.cluster_get(
cluster_id=install_env.get_cluster_id())
else:
cluster = client.create_cluster(cluster_name,
ssh_public_key=args.ssh_key,
**_cluster_create_params())
image_path = os.path.join(consts.IMAGE_FOLDER,
f'{args.namespace}-installer-image.iso')
if args.with_static_network_config:
tf_folder = utils.get_tf_folder(cluster_name, args.namespace)
static_network_config = static_network.generate_static_network_data_from_tf(
tf_folder)
else:
static_network_config = None
if image_url is not None:
utils.download_iso(image_url, image_path)
else:
client.generate_and_download_image(
cluster_id=cluster.id,
image_path=image_path,
image_type=image_type,
ssh_key=args.ssh_key,
static_network_config=static_network_config,
)
# Iso only, cluster will be up and iso downloaded but vm will not be created
if not args.iso_only:
try:
nodes_flow(client, cluster_name, cluster, machine_net, kube_client,
cluster_deployment)
finally:
if not image_path or args.keep_iso:
return
log.info('deleting iso: %s', image_path)
os.unlink(image_path)
return cluster.id