def wait_for_cluster_in_error_status(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.ERROR], timeout=consts.ERROR_TIMEOUT, )
def wait_for_pending_for_input_status(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.PENDING_FOR_INPUT], timeout=consts.PENDING_USER_ACTION_TIMEOUT, )
def wait_till_installed(client, cluster, timeout=60 * 60 * 2): log.info("Waiting %s till cluster finished installation", timeout) # TODO: Change host validation for only previous known hosts try: utils.wait_till_all_hosts_are_in_status( client=client, cluster_id=cluster.id, nodes_count=len(cluster.hosts), statuses=[consts.NodesStatus.INSTALLED], timeout=timeout, interval=60, ) utils.wait_till_cluster_is_in_status( client=client, cluster_id=cluster.id, statuses=[consts.ClusterStatus.INSTALLED], timeout=consts.CLUSTER_INSTALLATION_TIMEOUT if cluster.high_availability_mode == "Full" else consts.CLUSTER_INSTALLATION_TIMEOUT * 2, ) finally: output_folder = f'build/{cluster.id}' utils.recreate_folder(output_folder) download_logs_from_all_hosts(client=client, cluster_id=cluster.id, output_folder=output_folder)
def wait_for_cluster_to_be_in_installing_pending_user_action_status(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.INSTALLING_PENDING_USER_ACTION], timeout=consts.PENDING_USER_ACTION_TIMEOUT, )
def wait_for_cluster_to_be_in_status(self, statuses, timeout=consts.ERROR_TIMEOUT): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=statuses, timeout=timeout, )
def wait_till_installed(client, cluster, timeout=60 * 60 * 2): # TODO: Change host validation for only previous known hosts try: utils.wait_till_all_hosts_are_in_status( client=client, cluster_id=cluster.id, nodes_count=len(cluster.hosts), statuses=[consts.NodesStatus.INSTALLED], timeout=timeout, interval=60, ) utils.wait_till_all_operators_are_in_status( client=client, cluster_id=cluster.id, operators_count=len(cluster.monitored_operators), operator_types=[OperatorType.BUILTIN, OperatorType.OLM], statuses=[consts.OperatorStatus.AVAILABLE, consts.OperatorStatus.FAILED], timeout=consts.CLUSTER_INSTALLATION_TIMEOUT, fall_on_error_status=False, ) utils.wait_till_cluster_is_in_status( client=client, cluster_id=cluster.id, statuses=[consts.ClusterStatus.INSTALLED], timeout=consts.CLUSTER_INSTALLATION_TIMEOUT if cluster.high_availability_mode == "Full" else consts.CLUSTER_INSTALLATION_TIMEOUT * 2, break_statuses=[consts.ClusterStatus.ERROR] ) finally: output_folder = f'build/{cluster.id}' utils.recreate_folder(output_folder) download_logs_from_all_hosts(client=client, cluster_id=cluster.id, output_folder=output_folder)
def wait_for_cluster_to_be_in_installing_status(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.INSTALLING], timeout=consts.START_CLUSTER_INSTALLATION_TIMEOUT, )
def start_cluster_installation_rest_api( client, cluster, cluster_id, pull_secret, kubeconfig_path ): log.info("Verifying pull secret") verify_pull_secret( client=client, cluster=cluster, pull_secret=pull_secret, ) log.info("Wait till cluster is ready") utils.wait_till_cluster_is_in_status( client=client, cluster_id=cluster_id, statuses=[consts.ClusterStatus.READY, consts.ClusterStatus.INSTALLING], break_statuses=[consts.ClusterStatus.ERROR], ) cluster = client.cluster_get(cluster_id) if cluster.status == consts.ClusterStatus.READY: log.info("Install cluster %s", cluster_id) _install_cluster(client=client, cluster=cluster) else: log.info( "Cluster is already in installing status, skipping install command" ) log.info("Download kubeconfig-noingress") client.download_kubeconfig_no_ingress( cluster_id=cluster_id, kubeconfig_path=kubeconfig_path, )
def wait_for_install(self, timeout=consts.CLUSTER_INSTALLATION_TIMEOUT): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.INSTALLED], timeout=timeout, )
def wait_for_cluster_to_be_in_finalizing_status(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.FINALIZING, consts.ClusterStatus.INSTALLED], timeout=consts.CLUSTER_INSTALLATION_TIMEOUT, break_statuses=[consts.ClusterStatus.ERROR], )
def wait_until_cluster_starts_installation(cluster_id, api_client): utils.wait_till_cluster_is_in_status( client=api_client, cluster_id=cluster_id, statuses=[ consts.ClusterStatus.PREPARING_FOR_INSTALLATION, consts.ClusterStatus.INSTALLING, consts.ClusterStatus.FINALIZING ])
def wait_for_ready_to_install(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.READY]) # This code added due to BZ:1909997, temporarily checking if help to prevent unexpected failure time.sleep(90) utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.READY])
def run_install_flow(client, cluster_id, kubeconfig_path, pull_secret, tf=None, kube_client=None, cluster_deployment=None): log.info("Verifying cluster exists") cluster = client.cluster_get(cluster_id) log.info("Verifying pull secret") verify_pull_secret(client=client, cluster=cluster, pull_secret=pull_secret) log.info("Wait till cluster is ready") utils.wait_till_cluster_is_in_status( client=client, cluster_id=cluster_id, statuses=[consts.ClusterStatus.READY, consts.ClusterStatus.INSTALLING], break_statuses=[consts.ClusterStatus.ERROR] ) cluster = client.cluster_get(cluster_id) if cluster.status == consts.ClusterStatus.READY: log.info("Install cluster %s", cluster_id) _install_cluster(client=client, cluster=cluster) else: log.info("Cluster is already in installing status, skipping install command") log.info("Download kubeconfig-noingress") client.download_kubeconfig_no_ingress( cluster_id=cluster_id, kubeconfig_path=kubeconfig_path ) log.info("Waiting until cluster finishes installation") if kube_client is not None: cluster_deployment.wait_to_be_installed() else: wait_till_installed(client=client, cluster=cluster) log.info("Download kubeconfig") if kube_client is not None: cluster_deployment.download_kubeconfig(kubeconfig_path=kubeconfig_path) else: waiting.wait( lambda: client.download_kubeconfig( cluster_id=cluster_id, kubeconfig_path=kubeconfig_path ) is None, timeout_seconds=240, sleep_seconds=20, expected_exceptions=Exception, waiting_for="Kubeconfig", ) # set new vips if tf and kube_client is None: cluster_info = client.cluster_get(cluster.id) if not cluster_info.api_vip: cluster_info.api_vip = helper_cluster.get_api_vip_from_cluster(client, cluster_info) tf.set_new_vip(cluster_info.api_vip)
def run_install_flow(client, cluster_id, kubeconfig_path, pull_secret, tf=None): log.info("Verifying cluster exists") cluster = client.cluster_get(cluster_id) client.update_cluster_install_config( cluster_id, {"networking": { "networkType": "OpenShiftSDN" }}) log.info("Verifying pull secret") verify_pull_secret(client=client, cluster=cluster, pull_secret=pull_secret) log.info("Wait till cluster is ready") utils.wait_till_cluster_is_in_status( client=client, cluster_id=cluster_id, statuses=[consts.ClusterStatus.READY, consts.ClusterStatus.INSTALLING], ) cluster = client.cluster_get(cluster_id) if cluster.status == consts.ClusterStatus.READY: log.info("Install cluster %s", cluster_id) _install_cluster(client=client, cluster=cluster) else: log.info( "Cluster is already in installing status, skipping install command" ) log.info("Download kubeconfig-noingress") client.download_kubeconfig_no_ingress(cluster_id=cluster_id, kubeconfig_path=kubeconfig_path) wait_till_installed(client=client, cluster=cluster) log.info("Download kubeconfig") waiting.wait( lambda: client.download_kubeconfig( cluster_id=cluster_id, kubeconfig_path=kubeconfig_path) is None, timeout_seconds=240, sleep_seconds=20, expected_exceptions=Exception, waiting_for="Kubeconfig", ) # set new vips if tf: cluster_info = client.cluster_get(cluster.id) tf.set_new_vip(cluster_info.api_vip)
def _install_cluster(client, cluster): cluster = client.install_cluster(cluster_id=cluster.id) utils.wait_till_cluster_is_in_status( client=client, cluster_id=cluster.id, timeout=consts.START_CLUSTER_INSTALLATION_TIMEOUT, statuses=[consts.ClusterStatus.INSTALLING], ) utils.wait_till_all_hosts_are_in_status( client=client, cluster_id=cluster.id, nodes_count=len(cluster.hosts), statuses=[ consts.NodesStatus.INSTALLING, consts.NodesStatus.INSTALLING_IN_PROGRESS, ], interval=30, )
def wait_for_pending_for_input_status(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.PENDING_FOR_INPUT])
def wait_for_cluster_to_be_in_installing_status(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.INSTALLING])
def wait_until_cluster_is_ready_for_install(cluster_id, api_client): utils.wait_till_cluster_is_in_status( client=api_client, cluster_id=cluster_id, statuses=[consts.ClusterStatus.READY])
def wait_for_ready_to_install(self): utils.wait_till_cluster_is_in_status( client=self.api_client, cluster_id=self.id, statuses=[consts.ClusterStatus.READY])