def get_cluster_func(cluster_config: ClusterConfig = ClusterConfig()): if not cluster_config.cluster_name: cluster_config.cluster_name = env_variables.get( 'cluster_name', infra_utils.get_random_name(length=10)) res = Cluster(api_client=api_client, config=cluster_config) clusters.append(res) return res
def update_config(tf_config: TerraformConfig = TerraformConfig(), cluster_config: ClusterConfig = ClusterConfig(), operators=None): if operators is None: operators = parse_olm_operators_from_env() tf_config.worker_memory = resource_param(tf_config.worker_memory, OperatorResource.WORKER_MEMORY_KEY, operators) tf_config.master_memory = resource_param(tf_config.master_memory, OperatorResource.MASTER_MEMORY_KEY, operators) tf_config.worker_vcpu = resource_param(tf_config.worker_vcpu, OperatorResource.WORKER_VCPU_KEY, operators) tf_config.master_vcpu = resource_param(tf_config.master_vcpu, OperatorResource.MASTER_VCPU_KEY, operators) tf_config.workers_count = resource_param(tf_config.workers_count, OperatorResource.WORKER_COUNT_KEY, operators) tf_config.worker_disk = resource_param(tf_config.worker_disk, OperatorResource.WORKER_DISK_KEY, operators) tf_config.master_disk = resource_param(tf_config.master_disk, OperatorResource.MASTER_DISK_KEY, operators) tf_config.master_disk_count = resource_param(tf_config.master_disk_count, OperatorResource.MASTER_DISK_COUNT_KEY, operators) tf_config.worker_disk_count = resource_param(tf_config.worker_disk_count, OperatorResource.WORKER_DISK_COUNT_KEY, operators) cluster_config.workers_count = resource_param(cluster_config.workers_count, OperatorResource.WORKER_COUNT_KEY, operators) cluster_config.nodes_count = cluster_config.masters_count + cluster_config.workers_count cluster_config.olm_operators = [operators]
def new_cluster_configuration(self, request: FixtureRequest): # Overriding the default BaseTest.new_cluster_configuration fixture to set custom configs. config = ClusterConfig() for fixture_name in [ "openshift_version", "network_type", "is_static_ip" ]: with suppress(FixtureLookupError): setattr(config, fixture_name, request.getfixturevalue(fixture_name)) config.trigger() return config
def prepare_worker_installation( self, controller_configuration: BaseNodeConfig, cluster_configuration: ClusterConfig, master_image_path: str, ): cluster_configuration.iso_download_path = master_image_path with open(os.path.join(IBIP_DIR, "worker-live-iso.ign"), "w") as f: f.write(self.render_worker_live_iso_ignition(INSTALLATION_DISK)) worker_image_path = self.embed("installer-image.iso", "worker-live-iso.ign", EMBED_IMAGE_NAME_WORKER) cluster_configuration.worker_iso_download_path = worker_image_path
def test_kube_api_ipv4(self, kube_api_context, get_nodes): tf_config = TerraformConfig(masters_count=1, workers_count=0, master_vcpu=8, master_memory=35840) cluster_config = ClusterConfig() kube_api_test(kube_api_context, get_nodes(tf_config), cluster_config)
def new_cluster_configuration(self) -> ClusterConfig: """ Creates new cluster configuration object. Override this fixture in your test class to provide a custom cluster configuration. (See TestInstall) :rtype: new cluster configuration object """ return ClusterConfig()
def get_api_vip_from_cluster(api_client, cluster_info: Union[dict, models.cluster.Cluster], pull_secret): import warnings from tests.config import ClusterConfig warnings.warn( "Soon get_api_vip_from_cluster will be deprecated. Avoid using or adding new functionality to " "this function. The function and solution for that case have not been determined yet. It might be " "on another module, or as a classmethod within Cluster class." " For more information see https://issues.redhat.com/browse/MGMT-4975", PendingDeprecationWarning, ) if isinstance(cluster_info, dict): cluster_info = models.cluster.Cluster(**cluster_info) cluster = Cluster( api_client=api_client, config=ClusterConfig( cluster_name=ClusterName(cluster_info.name), pull_secret=pull_secret, ssh_public_key=cluster_info.ssh_public_key, cluster_id=cluster_info.id), nodes=None ) return cluster.get_api_vip(cluster=cluster_info)
def cluster( self, api_client: InventoryClient, request: FixtureRequest, infra_env_configuration: InfraEnvConfig, proxy_server, prepare_nodes_network: Nodes, cluster_configuration: ClusterConfig, ipxe_server: Callable, tang_server: Callable, ): log.debug( f"--- SETUP --- Creating cluster for test: {request.node.name}\n") if cluster_configuration.disk_encryption_mode == consts.DiskEncryptionMode.TANG: self._start_tang_server(tang_server, cluster_configuration) cluster = Cluster( api_client=api_client, config=cluster_configuration, infra_env_config=infra_env_configuration, nodes=prepare_nodes_network, ) if self._does_need_proxy_server(prepare_nodes_network): self.__set_up_proxy_server(cluster, cluster_configuration, proxy_server) if global_variables.ipxe_boot: infra_env = cluster.generate_infra_env() ipxe_server_controller = ipxe_server(name="ipxe_controller", api_client=cluster.api_client) ipxe_server_controller.run(infra_env_id=infra_env.id, cluster_name=cluster.name) cluster_configuration.iso_download_path = utils.get_iso_download_path( infra_env_configuration.entity_name.get()) yield cluster if self._is_test_failed(request): log.info( f"--- TEARDOWN --- Collecting Logs for test: {request.node.name}\n" ) self.collect_test_logs(cluster, api_client, request, cluster.nodes) if global_variables.test_teardown: if cluster.is_installing() or cluster.is_finalizing(): cluster.cancel_install() if global_variables.test_teardown: with SuppressAndLog(ApiException): cluster.deregister_infraenv() with suppress(ApiException): log.info( f"--- TEARDOWN --- deleting created cluster {cluster.id}\n" ) cluster.delete()
def gather_sosreport_data(output_dir: str): sosreport_output = os.path.join(output_dir, "sosreport") recreate_folder(sosreport_output) controller = LibvirtController(config=TerraformConfig(), entity_config=ClusterConfig()) run_concurrently( jobs=[(gather_sosreport_from_node, node, sosreport_output) for node in controller.list_nodes()], timeout=60 * 20, )
def test_olm_operator(self, get_nodes, get_cluster, operators, update_olm_config): new_cluster = get_cluster( cluster_config=ClusterConfig(olm_operators=[operators]), nodes=get_nodes( update_olm_config(config=TerraformConfig(), operators=operators))) new_cluster.prepare_for_installation() new_cluster.start_install_and_wait_for_installed() assert new_cluster.is_operator_in_status(operators, OperatorStatus.AVAILABLE)
def new_cluster_configuration(self, request: FixtureRequest) -> ClusterConfig: """ Creates new cluster configuration object. Override this fixture in your test class to provide a custom cluster configuration. (See TestInstall) :rtype: new cluster configuration object """ config = ClusterConfig() self.update_parameterized(request, config) return config
def test_bootstrap_in_place_sno( self, controller: NodeController, controller_configuration: BaseNodeConfig, cluster_configuration: ClusterConfig ): image_path = self.prepare_installation(controller_configuration, cluster_configuration) log.info("Starting node...") cluster_configuration.iso_download_path = image_path controller.start_all_nodes() log.info("Node started!") controller.start_all_nodes() self.waiting_for_installation_completion(controller, cluster_configuration)
def create_controller(net_asset): return TerraformController( TerraformConfig( masters_count=1, workers_count=0, master_memory=45 * 1024, # in megabytes master_vcpu=16, net_asset=net_asset, bootstrap_in_place=True, single_node_ip=net_asset.machine_cidr.replace("0/24", "10"), ), entity_config=ClusterConfig( cluster_name=ClusterName(prefix="test-infra-cluster", suffix="")))
def _start_tang_server(cls, tang_server: Callable, cluster_configuration: ClusterConfig, server_name: str = "tang1"): new_tang_server = tang_server( name=server_name, port=consts.DEFAULT_TANG_SERVER_PORT, pull_secret=cluster_configuration.pull_secret) new_tang_server.run() new_tang_server.set_thumbprint() cluster_configuration.tang_servers = ( f'[{{"url":"{new_tang_server.address}","thumbprint":"{new_tang_server.thumbprint}"}}]' )
def test_delete_clusters(self, api_client: InventoryClient, cluster_configuration): """Delete all clusters or single cluster if CLUSTER_ID is given""" cluster_id = cluster_configuration.cluster_id clusters = api_client.clusters_list() if not cluster_id else [{ "id": cluster_id }] for cluster_info in clusters: cluster = Cluster(api_client, ClusterConfig(cluster_id=cluster_info["id"]), InfraEnvConfig()) cluster.delete() log.info(f"Successfully deleted {len(clusters)} clusters")
def new_cluster_configuration(self, request: FixtureRequest): # Overriding the default BaseTest.new_cluster_configuration fixture to set custom configs. config = ClusterConfig() for fixture_name in ["openshift_version", "network_type", "is_static_ip", "olm_operators"]: with suppress(FixtureLookupError): if hasattr(config, fixture_name): config.set_value(fixture_name, request.getfixturevalue(fixture_name)) else: raise AttributeError(f"No attribute name {fixture_name} in ClusterConfig object type") config.trigger(get_default_triggers()) return config
def test_kube_api_ipv6(self, kube_api_context, proxy_server, get_nodes): tf_config = TerraformConfig(masters_count=1, workers_count=0, master_vcpu=8, master_memory=35840, is_ipv6=True) cluster_config = ClusterConfig( service_network_cidr='2003:db8::/112', cluster_network_cidr='2002:db8::/53', cluster_network_host_prefix=64, is_ipv6=True, ) kube_api_test(kube_api_context, get_nodes(tf_config), cluster_config, proxy_server, is_ipv4=False)
def configs(self) -> Tuple[ClusterConfig, TerraformConfig]: """ Get configurations objects - while using configs fixture cluster and tf configs are the same For creating new Config object just call it explicitly e.g. ClusterConfig(masters_count=1) """ yield ClusterConfig(), TerraformConfig()
def test_olm_operator(self, get_nodes, get_cluster, olm_operator): new_cluster = get_cluster(cluster_config=ClusterConfig(olm_operators=[olm_operator]), nodes=get_nodes(TerraformConfig(olm_operators=[olm_operator]))) new_cluster.prepare_for_installation() new_cluster.start_install_and_wait_for_installed()
def override_cluster_configuration(self): config = ClusterConfig() config.cluster_id = global_variables.cluster_id return config
def kube_api_test( self, kube_api_context: KubeAPIContext, nodes: Nodes, cluster_config: ClusterConfig, prepared_controller_configuration: BaseNodeConfig, infra_env_configuration: BaseInfraEnvConfig, proxy_server: Optional[Callable] = None, *, is_disconnected: bool = False, ): cluster_name = cluster_config.cluster_name.get() api_client = kube_api_context.api_client spoke_namespace = kube_api_context.spoke_namespace # TODO resolve it from the service if the node controller doesn't have this information # (please see cluster.get_primary_machine_cidr()) agent_cluster_install = AgentClusterInstall( api_client, f"{cluster_name}-agent-cluster-install", spoke_namespace ) secret = Secret(api_client, f"{cluster_name}-secret", spoke_namespace) secret.create(pull_secret=cluster_config.pull_secret) cluster_deployment = ClusterDeployment(api_client, cluster_name, spoke_namespace) cluster_deployment.create(agent_cluster_install_ref=agent_cluster_install.ref, secret=secret) proxy = self.setup_proxy(nodes, cluster_config, proxy_server) if is_disconnected: log.info("getting ignition and install config override for disconnected install") ca_bundle = self.get_ca_bundle_from_hub(spoke_namespace) self.patch_install_config_with_ca_bundle(cluster_deployment, ca_bundle) ignition_config_override = self.get_ignition_config_override(ca_bundle) else: ignition_config_override = None infra_env = InfraEnv(api_client, f"{cluster_name}-infra-env", spoke_namespace) infraenv = infra_env.create( cluster_deployment, secret, proxy, ignition_config_override, ssh_pub_key=cluster_config.ssh_public_key ) cluster_config.iso_download_path = utils.get_iso_download_path(infraenv.get("metadata", {}).get("name")) nodes.prepare_nodes() agent_cluster_install.create( cluster_deployment_ref=cluster_deployment.ref, image_set_ref=self.deploy_image_set(cluster_name, api_client), cluster_cidr=cluster_config.cluster_networks[0].cidr, host_prefix=cluster_config.cluster_networks[0].host_prefix, service_network=cluster_config.service_networks[0].cidr, ssh_pub_key=cluster_config.ssh_public_key, hyperthreading=cluster_config.hyperthreading, control_plane_agents=nodes.masters_count, worker_agents=nodes.workers_count, proxy=proxy.as_dict() if proxy else {}, ) agent_cluster_install.wait_to_be_ready(ready=False) if infra_env_configuration.is_static_ip: self.apply_static_network_config(kube_api_context, nodes, cluster_name) agents = self.start_nodes(nodes, infra_env, cluster_config, infra_env_configuration.is_static_ip) if len(nodes) == 1: # for single node set the cidr and take the actual ip from the host # the vips is the ip of the host self._set_agent_cluster_install_machine_cidr(agent_cluster_install, nodes) # wait till the ip is set for the node and read it from its inventory self.set_single_node_ip(cluster_deployment, nodes) api_vip = ingress_vip = get_ip_for_single_node(cluster_deployment, nodes.is_ipv4) else: # for multi node allocate 2 address at a safe distance from the beginning # of the available address block to allow enough addresses for workers access_vips = nodes.controller.get_ingress_and_api_vips() api_vip = access_vips["api_vip"] ingress_vip = access_vips["ingress_vip"] # patch the aci with the vips. The cidr will be derived from the range agent_cluster_install.set_api_vip(api_vip) agent_cluster_install.set_ingress_vip(ingress_vip) nodes.controller.set_dns(api_ip=api_vip, ingress_ip=ingress_vip) log.info("Waiting for install") self._wait_for_install(agent_cluster_install, agents)
def capi_test( self, kube_api_context: KubeAPIContext, nodes: Nodes, cluster_config: ClusterConfig, is_static_ip: bool, proxy_server: Optional[Callable] = None, *, is_disconnected: bool = False, ): cluster_name = cluster_config.cluster_name.get() api_client = kube_api_context.api_client spoke_namespace = kube_api_context.spoke_namespace cluster_config.iso_download_path = utils.get_iso_download_path(cluster_name) nodes.prepare_nodes() secret = Secret(api_client, f"{cluster_name}-secret", spoke_namespace) secret.create(pull_secret=cluster_config.pull_secret) if is_disconnected: log.info("getting igntion and install config override for disconected install") ca_bundle = self.get_ca_bundle_from_hub(spoke_namespace) ignition_config_override = self.get_ignition_config_override(ca_bundle) else: ignition_config_override = None proxy = self.setup_proxy(nodes, cluster_config, proxy_server) infra_env = InfraEnv(api_client, f"{cluster_name}-infra-env", spoke_namespace) infra_env.create( cluster_deployment=None, ignition_config_override=ignition_config_override, secret=secret, proxy=proxy, ssh_pub_key=cluster_config.ssh_public_key, ) self.start_nodes(nodes, infra_env, cluster_config, is_static_ip) hypershift = HyperShift(name=cluster_name, kube_api_client=api_client) with utils.pull_secret_file() as ps: with tempfile.NamedTemporaryFile(mode="w") as f: f.write(cluster_config.ssh_public_key) f.flush() ssh_public_key_file = f.name hypershift.create( pull_secret_file=ps, agent_namespace=spoke_namespace, provider_image=os.environ.get("PROVIDER_IMAGE", ""), hypershift_cpo_image=os.environ.get("HYPERSHIFT_IMAGE", ""), release_image=os.environ.get("OPENSHIFT_INSTALL_RELEASE_IMAGE", ""), ssh_key=ssh_public_key_file, ) hypershift.wait_for_control_plane_ready() # WORKAROUND for ovn on minikube secret = Secret(api_client, "ovn-master-metrics-cert", hypershift.namespace) secret.create_with_data(secret_data={"ca_cert": "dummy data, we only need this secret to exists"}) cluster_deployment = ClusterDeployment(api_client, cluster_name, f"clusters-{cluster_name}") def _cluster_deployment_installed() -> bool: return cluster_deployment.get().get("spec", {}).get("installed") waiting.wait( _cluster_deployment_installed, sleep_seconds=1, timeout_seconds=60, waiting_for="clusterDeployment to get created", expected_exceptions=Exception, ) hypershift.wait_for_control_plane_ready() self.set_node_count_and_wait_for_ready_nodes(cluster_deployment, hypershift, spoke_namespace, node_count=1) self.set_node_count_and_wait_for_ready_nodes(cluster_deployment, hypershift, spoke_namespace, node_count=2) self.scale_down_nodepool_and_wait_for_unbounded_agent( cluster_deployment, hypershift, spoke_namespace, node_count=1 )
def cluster_config(self) -> ClusterConfig: yield ClusterConfig()
def test_install(self, get_nodes, get_cluster, openshift_version): new_cluster = get_cluster( cluster_config=ClusterConfig(openshift_version=openshift_version), nodes=get_nodes()) new_cluster.prepare_for_installation() new_cluster.start_install_and_wait_for_installed()
def test_install(self, nodes: Nodes, cluster, openshift_version): new_cluster = cluster(cluster_config=ClusterConfig( openshift_version=openshift_version)) new_cluster.prepare_for_installation(nodes) new_cluster.start_install_and_wait_for_installed()
def test_olm_operator(self, nodes: Nodes, cluster, olm_operator): new_cluster = cluster(cluster_config=ClusterConfig( olm_operators=[olm_operator])) new_cluster.prepare_for_installation(nodes, olm_operators=[olm_operator]) new_cluster.start_install_and_wait_for_installed()
def start_install_and_wait_for_installed(self): cluster_name = self.config.day1_cluster_name # Running twice as a workaround for an issue with terraform not spawning a new node on first apply. for _ in range(2): with utils.file_lock_context(): utils.run_command( f"make _apply_terraform CLUSTER_NAME={cluster_name} PLATFORM={consts.Platforms.BARE_METAL}" ) time.sleep(5) num_nodes_to_wait = self.config.day2_workers_count installed_status = consts.NodesStatus.DAY2_INSTALLED tfvars = utils.get_tfvars(self.config.tf_folder) tf_network_name = tfvars["libvirt_network_name"] config = TerraformConfig() config.nodes_count = num_nodes_to_wait libvirt_controller = LibvirtController(config=config, entity_config=ClusterConfig()) libvirt_controller.wait_till_nodes_are_ready( network_name=tf_network_name) # Wait for day2 nodes waiting.wait( lambda: self.are_libvirt_nodes_in_cluster_hosts(), timeout_seconds=consts.NODES_REGISTERED_TIMEOUT, sleep_seconds=10, waiting_for="Nodes to be registered in inventory service", ) self.set_nodes_hostnames_if_needed(tf_network_name) wait_till_all_hosts_are_in_status( client=self.api_client, cluster_id=self.config.cluster_id, nodes_count=self.config.day2_workers_count, statuses=[consts.NodesStatus.KNOWN], interval=30, ) # Start day2 nodes installation log.info("Start installing all known nodes in the cluster %s", self.config.cluster_id) kubeconfig = utils.get_kubeconfig_path(self.config.day1_cluster_name) ocp_ready_nodes = self.get_ocp_cluster_ready_nodes_num(kubeconfig) hosts = self.api_client.get_cluster_hosts(self.config.cluster_id) [ self.api_client.install_day2_host(self.config.infra_env_id, host["id"]) for host in hosts if host["status"] == "known" ] log.info( "Waiting until all nodes of cluster %s have been installed (reached added-to-existing-cluster)", self.config.cluster_id, ) wait_till_all_hosts_are_in_status( client=self.api_client, cluster_id=self.config.cluster_id, nodes_count=num_nodes_to_wait, statuses=[installed_status], interval=30, ) log.info( "Waiting until installed nodes has actually been added to the OCP cluster" ) waiting.wait( lambda: self.wait_nodes_join_ocp_cluster( ocp_ready_nodes, self.config.day2_workers_count, kubeconfig), timeout_seconds=consts.NODES_REGISTERED_TIMEOUT, sleep_seconds=30, waiting_for="Day2 nodes to be added to OCP cluster", expected_exceptions=Exception, ) log.info("%d worker nodes were successfully added to OCP cluster", self.config.day2_workers_count)
def new_cluster_configuration(self, request) -> ClusterConfig: return ClusterConfig( cluster_name=ClusterName(prefix=CLUSTER_PREFIX, suffix=""))