def get_cluster_func(cluster_config: ClusterConfig = ClusterConfig()):
if not cluster_config.cluster_name:
cluster_config.cluster_name = env_variables.get(
'cluster_name', infra_utils.get_random_name(length=10))
res = Cluster(api_client=api_client, config=cluster_config)
clusters.append(res)
return res
def update_config(tf_config: TerraformConfig = TerraformConfig(),
cluster_config: ClusterConfig = ClusterConfig(), operators=None):
if operators is None:
operators = parse_olm_operators_from_env()
tf_config.worker_memory = resource_param(tf_config.worker_memory,
OperatorResource.WORKER_MEMORY_KEY, operators)
tf_config.master_memory = resource_param(tf_config.master_memory,
OperatorResource.MASTER_MEMORY_KEY, operators)
tf_config.worker_vcpu = resource_param(tf_config.worker_vcpu,
OperatorResource.WORKER_VCPU_KEY, operators)
tf_config.master_vcpu = resource_param(tf_config.master_vcpu,
OperatorResource.MASTER_VCPU_KEY, operators)
tf_config.workers_count = resource_param(tf_config.workers_count,
OperatorResource.WORKER_COUNT_KEY, operators)
tf_config.worker_disk = resource_param(tf_config.worker_disk,
OperatorResource.WORKER_DISK_KEY, operators)
tf_config.master_disk = resource_param(tf_config.master_disk,
OperatorResource.MASTER_DISK_KEY, operators)
tf_config.master_disk_count = resource_param(tf_config.master_disk_count,
OperatorResource.MASTER_DISK_COUNT_KEY, operators)
tf_config.worker_disk_count = resource_param(tf_config.worker_disk_count,
OperatorResource.WORKER_DISK_COUNT_KEY, operators)
cluster_config.workers_count = resource_param(cluster_config.workers_count,
OperatorResource.WORKER_COUNT_KEY, operators)
cluster_config.nodes_count = cluster_config.masters_count + cluster_config.workers_count
cluster_config.olm_operators = [operators]
def new_cluster_configuration(self, request: FixtureRequest):
# Overriding the default BaseTest.new_cluster_configuration fixture to set custom configs.
config = ClusterConfig()
for fixture_name in [
"openshift_version", "network_type", "is_static_ip"
]:
with suppress(FixtureLookupError):
setattr(config, fixture_name,
request.getfixturevalue(fixture_name))
config.trigger()
return config
def prepare_worker_installation(
self,
controller_configuration: BaseNodeConfig,
cluster_configuration: ClusterConfig,
master_image_path: str,
):
cluster_configuration.iso_download_path = master_image_path
with open(os.path.join(IBIP_DIR, "worker-live-iso.ign"), "w") as f:
f.write(self.render_worker_live_iso_ignition(INSTALLATION_DISK))
worker_image_path = self.embed("installer-image.iso", "worker-live-iso.ign", EMBED_IMAGE_NAME_WORKER)
cluster_configuration.worker_iso_download_path = worker_image_path
def test_kube_api_ipv4(self, kube_api_context, get_nodes):
tf_config = TerraformConfig(masters_count=1,
workers_count=0,
master_vcpu=8,
master_memory=35840)
cluster_config = ClusterConfig()
kube_api_test(kube_api_context, get_nodes(tf_config), cluster_config)
def new_cluster_configuration(self) -> ClusterConfig:
"""
Creates new cluster configuration object.
Override this fixture in your test class to provide a custom cluster configuration. (See TestInstall)
:rtype: new cluster configuration object
"""
return ClusterConfig()
def get_api_vip_from_cluster(api_client, cluster_info: Union[dict, models.cluster.Cluster], pull_secret):
import warnings
from tests.config import ClusterConfig
warnings.warn(
"Soon get_api_vip_from_cluster will be deprecated. Avoid using or adding new functionality to "
"this function. The function and solution for that case have not been determined yet. It might be "
"on another module, or as a classmethod within Cluster class."
" For more information see https://issues.redhat.com/browse/MGMT-4975",
PendingDeprecationWarning,
)
if isinstance(cluster_info, dict):
cluster_info = models.cluster.Cluster(**cluster_info)
cluster = Cluster(
api_client=api_client,
config=ClusterConfig(
cluster_name=ClusterName(cluster_info.name),
pull_secret=pull_secret,
ssh_public_key=cluster_info.ssh_public_key,
cluster_id=cluster_info.id),
nodes=None
)
return cluster.get_api_vip(cluster=cluster_info)
def cluster(
self,
api_client: InventoryClient,
request: FixtureRequest,
infra_env_configuration: InfraEnvConfig,
proxy_server,
prepare_nodes_network: Nodes,
cluster_configuration: ClusterConfig,
ipxe_server: Callable,
tang_server: Callable,
):
log.debug(
f"--- SETUP --- Creating cluster for test: {request.node.name}\n")
if cluster_configuration.disk_encryption_mode == consts.DiskEncryptionMode.TANG:
self._start_tang_server(tang_server, cluster_configuration)
cluster = Cluster(
api_client=api_client,
config=cluster_configuration,
infra_env_config=infra_env_configuration,
nodes=prepare_nodes_network,
)
if self._does_need_proxy_server(prepare_nodes_network):
self.__set_up_proxy_server(cluster, cluster_configuration,
proxy_server)
if global_variables.ipxe_boot:
infra_env = cluster.generate_infra_env()
ipxe_server_controller = ipxe_server(name="ipxe_controller",
api_client=cluster.api_client)
ipxe_server_controller.run(infra_env_id=infra_env.id,
cluster_name=cluster.name)
cluster_configuration.iso_download_path = utils.get_iso_download_path(
infra_env_configuration.entity_name.get())
yield cluster
if self._is_test_failed(request):
log.info(
f"--- TEARDOWN --- Collecting Logs for test: {request.node.name}\n"
)
self.collect_test_logs(cluster, api_client, request, cluster.nodes)
if global_variables.test_teardown:
if cluster.is_installing() or cluster.is_finalizing():
cluster.cancel_install()
if global_variables.test_teardown:
with SuppressAndLog(ApiException):
cluster.deregister_infraenv()
with suppress(ApiException):
log.info(
f"--- TEARDOWN --- deleting created cluster {cluster.id}\n"
)
cluster.delete()
def gather_sosreport_data(output_dir: str):
sosreport_output = os.path.join(output_dir, "sosreport")
recreate_folder(sosreport_output)
controller = LibvirtController(config=TerraformConfig(),
entity_config=ClusterConfig())
run_concurrently(
jobs=[(gather_sosreport_from_node, node, sosreport_output)
for node in controller.list_nodes()],
timeout=60 * 20,
)
def test_olm_operator(self, get_nodes, get_cluster, operators,
update_olm_config):
new_cluster = get_cluster(
cluster_config=ClusterConfig(olm_operators=[operators]),
nodes=get_nodes(
update_olm_config(config=TerraformConfig(),
operators=operators)))
new_cluster.prepare_for_installation()
new_cluster.start_install_and_wait_for_installed()
assert new_cluster.is_operator_in_status(operators,
OperatorStatus.AVAILABLE)
def new_cluster_configuration(self,
request: FixtureRequest) -> ClusterConfig:
"""
Creates new cluster configuration object.
Override this fixture in your test class to provide a custom cluster configuration. (See TestInstall)
:rtype: new cluster configuration object
"""
config = ClusterConfig()
self.update_parameterized(request, config)
return config
def test_bootstrap_in_place_sno(
self, controller: NodeController, controller_configuration: BaseNodeConfig, cluster_configuration: ClusterConfig
):
image_path = self.prepare_installation(controller_configuration, cluster_configuration)
log.info("Starting node...")
cluster_configuration.iso_download_path = image_path
controller.start_all_nodes()
log.info("Node started!")
controller.start_all_nodes()
self.waiting_for_installation_completion(controller, cluster_configuration)
def create_controller(net_asset):
return TerraformController(
TerraformConfig(
masters_count=1,
workers_count=0,
master_memory=45 * 1024, # in megabytes
master_vcpu=16,
net_asset=net_asset,
bootstrap_in_place=True,
single_node_ip=net_asset.machine_cidr.replace("0/24", "10"),
),
entity_config=ClusterConfig(
cluster_name=ClusterName(prefix="test-infra-cluster", suffix="")))
def _start_tang_server(cls,
tang_server: Callable,
cluster_configuration: ClusterConfig,
server_name: str = "tang1"):
new_tang_server = tang_server(
name=server_name,
port=consts.DEFAULT_TANG_SERVER_PORT,
pull_secret=cluster_configuration.pull_secret)
new_tang_server.run()
new_tang_server.set_thumbprint()
cluster_configuration.tang_servers = (
f'[{{"url":"{new_tang_server.address}","thumbprint":"{new_tang_server.thumbprint}"}}]'
)
def test_delete_clusters(self, api_client: InventoryClient,
cluster_configuration):
"""Delete all clusters or single cluster if CLUSTER_ID is given"""
cluster_id = cluster_configuration.cluster_id
clusters = api_client.clusters_list() if not cluster_id else [{
"id":
cluster_id
}]
for cluster_info in clusters:
cluster = Cluster(api_client,
ClusterConfig(cluster_id=cluster_info["id"]),
InfraEnvConfig())
cluster.delete()
log.info(f"Successfully deleted {len(clusters)} clusters")
def new_cluster_configuration(self, request: FixtureRequest):
# Overriding the default BaseTest.new_cluster_configuration fixture to set custom configs.
config = ClusterConfig()
for fixture_name in ["openshift_version", "network_type", "is_static_ip", "olm_operators"]:
with suppress(FixtureLookupError):
if hasattr(config, fixture_name):
config.set_value(fixture_name, request.getfixturevalue(fixture_name))
else:
raise AttributeError(f"No attribute name {fixture_name} in ClusterConfig object type")
config.trigger(get_default_triggers())
return config
def test_kube_api_ipv6(self, kube_api_context, proxy_server, get_nodes):
tf_config = TerraformConfig(masters_count=1,
workers_count=0,
master_vcpu=8,
master_memory=35840,
is_ipv6=True)
cluster_config = ClusterConfig(
service_network_cidr='2003:db8::/112',
cluster_network_cidr='2002:db8::/53',
cluster_network_host_prefix=64,
is_ipv6=True,
)
kube_api_test(kube_api_context,
get_nodes(tf_config),
cluster_config,
proxy_server,
is_ipv4=False)
def configs(self) -> Tuple[ClusterConfig, TerraformConfig]:
""" Get configurations objects - while using configs fixture cluster and tf configs are the same
For creating new Config object just call it explicitly e.g. ClusterConfig(masters_count=1) """
yield ClusterConfig(), TerraformConfig()
def test_olm_operator(self, get_nodes, get_cluster, olm_operator):
new_cluster = get_cluster(cluster_config=ClusterConfig(olm_operators=[olm_operator]),
nodes=get_nodes(TerraformConfig(olm_operators=[olm_operator])))
new_cluster.prepare_for_installation()
new_cluster.start_install_and_wait_for_installed()
def override_cluster_configuration(self):
config = ClusterConfig()
config.cluster_id = global_variables.cluster_id
return config
def kube_api_test(
self,
kube_api_context: KubeAPIContext,
nodes: Nodes,
cluster_config: ClusterConfig,
prepared_controller_configuration: BaseNodeConfig,
infra_env_configuration: BaseInfraEnvConfig,
proxy_server: Optional[Callable] = None,
*,
is_disconnected: bool = False,
):
cluster_name = cluster_config.cluster_name.get()
api_client = kube_api_context.api_client
spoke_namespace = kube_api_context.spoke_namespace
# TODO resolve it from the service if the node controller doesn't have this information
# (please see cluster.get_primary_machine_cidr())
agent_cluster_install = AgentClusterInstall(
api_client, f"{cluster_name}-agent-cluster-install", spoke_namespace
)
secret = Secret(api_client, f"{cluster_name}-secret", spoke_namespace)
secret.create(pull_secret=cluster_config.pull_secret)
cluster_deployment = ClusterDeployment(api_client, cluster_name, spoke_namespace)
cluster_deployment.create(agent_cluster_install_ref=agent_cluster_install.ref, secret=secret)
proxy = self.setup_proxy(nodes, cluster_config, proxy_server)
if is_disconnected:
log.info("getting ignition and install config override for disconnected install")
ca_bundle = self.get_ca_bundle_from_hub(spoke_namespace)
self.patch_install_config_with_ca_bundle(cluster_deployment, ca_bundle)
ignition_config_override = self.get_ignition_config_override(ca_bundle)
else:
ignition_config_override = None
infra_env = InfraEnv(api_client, f"{cluster_name}-infra-env", spoke_namespace)
infraenv = infra_env.create(
cluster_deployment, secret, proxy, ignition_config_override, ssh_pub_key=cluster_config.ssh_public_key
)
cluster_config.iso_download_path = utils.get_iso_download_path(infraenv.get("metadata", {}).get("name"))
nodes.prepare_nodes()
agent_cluster_install.create(
cluster_deployment_ref=cluster_deployment.ref,
image_set_ref=self.deploy_image_set(cluster_name, api_client),
cluster_cidr=cluster_config.cluster_networks[0].cidr,
host_prefix=cluster_config.cluster_networks[0].host_prefix,
service_network=cluster_config.service_networks[0].cidr,
ssh_pub_key=cluster_config.ssh_public_key,
hyperthreading=cluster_config.hyperthreading,
control_plane_agents=nodes.masters_count,
worker_agents=nodes.workers_count,
proxy=proxy.as_dict() if proxy else {},
)
agent_cluster_install.wait_to_be_ready(ready=False)
if infra_env_configuration.is_static_ip:
self.apply_static_network_config(kube_api_context, nodes, cluster_name)
agents = self.start_nodes(nodes, infra_env, cluster_config, infra_env_configuration.is_static_ip)
if len(nodes) == 1:
# for single node set the cidr and take the actual ip from the host
# the vips is the ip of the host
self._set_agent_cluster_install_machine_cidr(agent_cluster_install, nodes)
# wait till the ip is set for the node and read it from its inventory
self.set_single_node_ip(cluster_deployment, nodes)
api_vip = ingress_vip = get_ip_for_single_node(cluster_deployment, nodes.is_ipv4)
else:
# for multi node allocate 2 address at a safe distance from the beginning
# of the available address block to allow enough addresses for workers
access_vips = nodes.controller.get_ingress_and_api_vips()
api_vip = access_vips["api_vip"]
ingress_vip = access_vips["ingress_vip"]
# patch the aci with the vips. The cidr will be derived from the range
agent_cluster_install.set_api_vip(api_vip)
agent_cluster_install.set_ingress_vip(ingress_vip)
nodes.controller.set_dns(api_ip=api_vip, ingress_ip=ingress_vip)
log.info("Waiting for install")
self._wait_for_install(agent_cluster_install, agents)
def capi_test(
self,
kube_api_context: KubeAPIContext,
nodes: Nodes,
cluster_config: ClusterConfig,
is_static_ip: bool,
proxy_server: Optional[Callable] = None,
*,
is_disconnected: bool = False,
):
cluster_name = cluster_config.cluster_name.get()
api_client = kube_api_context.api_client
spoke_namespace = kube_api_context.spoke_namespace
cluster_config.iso_download_path = utils.get_iso_download_path(cluster_name)
nodes.prepare_nodes()
secret = Secret(api_client, f"{cluster_name}-secret", spoke_namespace)
secret.create(pull_secret=cluster_config.pull_secret)
if is_disconnected:
log.info("getting igntion and install config override for disconected install")
ca_bundle = self.get_ca_bundle_from_hub(spoke_namespace)
ignition_config_override = self.get_ignition_config_override(ca_bundle)
else:
ignition_config_override = None
proxy = self.setup_proxy(nodes, cluster_config, proxy_server)
infra_env = InfraEnv(api_client, f"{cluster_name}-infra-env", spoke_namespace)
infra_env.create(
cluster_deployment=None,
ignition_config_override=ignition_config_override,
secret=secret,
proxy=proxy,
ssh_pub_key=cluster_config.ssh_public_key,
)
self.start_nodes(nodes, infra_env, cluster_config, is_static_ip)
hypershift = HyperShift(name=cluster_name, kube_api_client=api_client)
with utils.pull_secret_file() as ps:
with tempfile.NamedTemporaryFile(mode="w") as f:
f.write(cluster_config.ssh_public_key)
f.flush()
ssh_public_key_file = f.name
hypershift.create(
pull_secret_file=ps,
agent_namespace=spoke_namespace,
provider_image=os.environ.get("PROVIDER_IMAGE", ""),
hypershift_cpo_image=os.environ.get("HYPERSHIFT_IMAGE", ""),
release_image=os.environ.get("OPENSHIFT_INSTALL_RELEASE_IMAGE", ""),
ssh_key=ssh_public_key_file,
)
hypershift.wait_for_control_plane_ready()
# WORKAROUND for ovn on minikube
secret = Secret(api_client, "ovn-master-metrics-cert", hypershift.namespace)
secret.create_with_data(secret_data={"ca_cert": "dummy data, we only need this secret to exists"})
cluster_deployment = ClusterDeployment(api_client, cluster_name, f"clusters-{cluster_name}")
def _cluster_deployment_installed() -> bool:
return cluster_deployment.get().get("spec", {}).get("installed")
waiting.wait(
_cluster_deployment_installed,
sleep_seconds=1,
timeout_seconds=60,
waiting_for="clusterDeployment to get created",
expected_exceptions=Exception,
)
hypershift.wait_for_control_plane_ready()
self.set_node_count_and_wait_for_ready_nodes(cluster_deployment, hypershift, spoke_namespace, node_count=1)
self.set_node_count_and_wait_for_ready_nodes(cluster_deployment, hypershift, spoke_namespace, node_count=2)
self.scale_down_nodepool_and_wait_for_unbounded_agent(
cluster_deployment, hypershift, spoke_namespace, node_count=1
)
def cluster_config(self) -> ClusterConfig:
yield ClusterConfig()
def test_install(self, get_nodes, get_cluster, openshift_version):
new_cluster = get_cluster(
cluster_config=ClusterConfig(openshift_version=openshift_version),
nodes=get_nodes())
new_cluster.prepare_for_installation()
new_cluster.start_install_and_wait_for_installed()
def test_install(self, nodes: Nodes, cluster, openshift_version):
new_cluster = cluster(cluster_config=ClusterConfig(
openshift_version=openshift_version))
new_cluster.prepare_for_installation(nodes)
new_cluster.start_install_and_wait_for_installed()
def test_olm_operator(self, nodes: Nodes, cluster, olm_operator):
new_cluster = cluster(cluster_config=ClusterConfig(
olm_operators=[olm_operator]))
new_cluster.prepare_for_installation(nodes,
olm_operators=[olm_operator])
new_cluster.start_install_and_wait_for_installed()
def start_install_and_wait_for_installed(self):
cluster_name = self.config.day1_cluster_name
# Running twice as a workaround for an issue with terraform not spawning a new node on first apply.
for _ in range(2):
with utils.file_lock_context():
utils.run_command(
f"make _apply_terraform CLUSTER_NAME={cluster_name} PLATFORM={consts.Platforms.BARE_METAL}"
)
time.sleep(5)
num_nodes_to_wait = self.config.day2_workers_count
installed_status = consts.NodesStatus.DAY2_INSTALLED
tfvars = utils.get_tfvars(self.config.tf_folder)
tf_network_name = tfvars["libvirt_network_name"]
config = TerraformConfig()
config.nodes_count = num_nodes_to_wait
libvirt_controller = LibvirtController(config=config,
entity_config=ClusterConfig())
libvirt_controller.wait_till_nodes_are_ready(
network_name=tf_network_name)
# Wait for day2 nodes
waiting.wait(
lambda: self.are_libvirt_nodes_in_cluster_hosts(),
timeout_seconds=consts.NODES_REGISTERED_TIMEOUT,
sleep_seconds=10,
waiting_for="Nodes to be registered in inventory service",
)
self.set_nodes_hostnames_if_needed(tf_network_name)
wait_till_all_hosts_are_in_status(
client=self.api_client,
cluster_id=self.config.cluster_id,
nodes_count=self.config.day2_workers_count,
statuses=[consts.NodesStatus.KNOWN],
interval=30,
)
# Start day2 nodes installation
log.info("Start installing all known nodes in the cluster %s",
self.config.cluster_id)
kubeconfig = utils.get_kubeconfig_path(self.config.day1_cluster_name)
ocp_ready_nodes = self.get_ocp_cluster_ready_nodes_num(kubeconfig)
hosts = self.api_client.get_cluster_hosts(self.config.cluster_id)
[
self.api_client.install_day2_host(self.config.infra_env_id,
host["id"]) for host in hosts
if host["status"] == "known"
]
log.info(
"Waiting until all nodes of cluster %s have been installed (reached added-to-existing-cluster)",
self.config.cluster_id,
)
wait_till_all_hosts_are_in_status(
client=self.api_client,
cluster_id=self.config.cluster_id,
nodes_count=num_nodes_to_wait,
statuses=[installed_status],
interval=30,
)
log.info(
"Waiting until installed nodes has actually been added to the OCP cluster"
)
waiting.wait(
lambda: self.wait_nodes_join_ocp_cluster(
ocp_ready_nodes, self.config.day2_workers_count, kubeconfig),
timeout_seconds=consts.NODES_REGISTERED_TIMEOUT,
sleep_seconds=30,
waiting_for="Day2 nodes to be added to OCP cluster",
expected_exceptions=Exception,
)
log.info("%d worker nodes were successfully added to OCP cluster",
self.config.day2_workers_count)
def new_cluster_configuration(self, request) -> ClusterConfig:
return ClusterConfig(
cluster_name=ClusterName(prefix=CLUSTER_PREFIX, suffix=""))