def wait_for_cluster_in_error_status(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.ERROR],
timeout=consts.ERROR_TIMEOUT,
)
def wait_for_pending_for_input_status(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.PENDING_FOR_INPUT],
timeout=consts.PENDING_USER_ACTION_TIMEOUT,
)
def wait_till_installed(client, cluster, timeout=60 * 60 * 2):
log.info("Waiting %s till cluster finished installation", timeout)
# TODO: Change host validation for only previous known hosts
try:
utils.wait_till_all_hosts_are_in_status(
client=client,
cluster_id=cluster.id,
nodes_count=len(cluster.hosts),
statuses=[consts.NodesStatus.INSTALLED],
timeout=timeout,
interval=60,
)
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster.id,
statuses=[consts.ClusterStatus.INSTALLED],
timeout=consts.CLUSTER_INSTALLATION_TIMEOUT
if cluster.high_availability_mode == "Full" else
consts.CLUSTER_INSTALLATION_TIMEOUT * 2,
)
finally:
output_folder = f'build/{cluster.id}'
utils.recreate_folder(output_folder)
download_logs_from_all_hosts(client=client,
cluster_id=cluster.id,
output_folder=output_folder)
def wait_for_cluster_to_be_in_installing_pending_user_action_status(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.INSTALLING_PENDING_USER_ACTION],
timeout=consts.PENDING_USER_ACTION_TIMEOUT,
)
def wait_for_cluster_to_be_in_status(self, statuses, timeout=consts.ERROR_TIMEOUT):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=statuses,
timeout=timeout,
)
def wait_till_installed(client, cluster, timeout=60 * 60 * 2):
# TODO: Change host validation for only previous known hosts
try:
utils.wait_till_all_hosts_are_in_status(
client=client,
cluster_id=cluster.id,
nodes_count=len(cluster.hosts),
statuses=[consts.NodesStatus.INSTALLED],
timeout=timeout,
interval=60,
)
utils.wait_till_all_operators_are_in_status(
client=client,
cluster_id=cluster.id,
operators_count=len(cluster.monitored_operators),
operator_types=[OperatorType.BUILTIN, OperatorType.OLM],
statuses=[consts.OperatorStatus.AVAILABLE, consts.OperatorStatus.FAILED],
timeout=consts.CLUSTER_INSTALLATION_TIMEOUT,
fall_on_error_status=False,
)
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster.id,
statuses=[consts.ClusterStatus.INSTALLED],
timeout=consts.CLUSTER_INSTALLATION_TIMEOUT if cluster.high_availability_mode == "Full"
else consts.CLUSTER_INSTALLATION_TIMEOUT * 2,
break_statuses=[consts.ClusterStatus.ERROR]
)
finally:
output_folder = f'build/{cluster.id}'
utils.recreate_folder(output_folder)
download_logs_from_all_hosts(client=client, cluster_id=cluster.id, output_folder=output_folder)
def wait_for_cluster_to_be_in_installing_status(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.INSTALLING],
timeout=consts.START_CLUSTER_INSTALLATION_TIMEOUT,
)
def start_cluster_installation_rest_api(
client,
cluster,
cluster_id,
pull_secret,
kubeconfig_path
):
log.info("Verifying pull secret")
verify_pull_secret(
client=client,
cluster=cluster,
pull_secret=pull_secret,
)
log.info("Wait till cluster is ready")
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster_id,
statuses=[consts.ClusterStatus.READY, consts.ClusterStatus.INSTALLING],
break_statuses=[consts.ClusterStatus.ERROR],
)
cluster = client.cluster_get(cluster_id)
if cluster.status == consts.ClusterStatus.READY:
log.info("Install cluster %s", cluster_id)
_install_cluster(client=client, cluster=cluster)
else:
log.info(
"Cluster is already in installing status, skipping install command"
)
log.info("Download kubeconfig-noingress")
client.download_kubeconfig_no_ingress(
cluster_id=cluster_id,
kubeconfig_path=kubeconfig_path,
)
def wait_for_install(self, timeout=consts.CLUSTER_INSTALLATION_TIMEOUT):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.INSTALLED],
timeout=timeout,
)
def wait_for_cluster_to_be_in_finalizing_status(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.FINALIZING, consts.ClusterStatus.INSTALLED],
timeout=consts.CLUSTER_INSTALLATION_TIMEOUT,
break_statuses=[consts.ClusterStatus.ERROR],
)
def wait_until_cluster_starts_installation(cluster_id, api_client):
utils.wait_till_cluster_is_in_status(
client=api_client,
cluster_id=cluster_id,
statuses=[
consts.ClusterStatus.PREPARING_FOR_INSTALLATION,
consts.ClusterStatus.INSTALLING,
consts.ClusterStatus.FINALIZING
])
def wait_for_ready_to_install(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.READY])
# This code added due to BZ:1909997, temporarily checking if help to prevent unexpected failure
time.sleep(90)
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.READY])
def run_install_flow(client, cluster_id, kubeconfig_path, pull_secret,
tf=None, kube_client=None, cluster_deployment=None):
log.info("Verifying cluster exists")
cluster = client.cluster_get(cluster_id)
log.info("Verifying pull secret")
verify_pull_secret(client=client, cluster=cluster, pull_secret=pull_secret)
log.info("Wait till cluster is ready")
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster_id,
statuses=[consts.ClusterStatus.READY, consts.ClusterStatus.INSTALLING],
break_statuses=[consts.ClusterStatus.ERROR]
)
cluster = client.cluster_get(cluster_id)
if cluster.status == consts.ClusterStatus.READY:
log.info("Install cluster %s", cluster_id)
_install_cluster(client=client, cluster=cluster)
else:
log.info("Cluster is already in installing status, skipping install command")
log.info("Download kubeconfig-noingress")
client.download_kubeconfig_no_ingress(
cluster_id=cluster_id, kubeconfig_path=kubeconfig_path
)
log.info("Waiting until cluster finishes installation")
if kube_client is not None:
cluster_deployment.wait_to_be_installed()
else:
wait_till_installed(client=client, cluster=cluster)
log.info("Download kubeconfig")
if kube_client is not None:
cluster_deployment.download_kubeconfig(kubeconfig_path=kubeconfig_path)
else:
waiting.wait(
lambda: client.download_kubeconfig(
cluster_id=cluster_id, kubeconfig_path=kubeconfig_path
) is None,
timeout_seconds=240,
sleep_seconds=20,
expected_exceptions=Exception,
waiting_for="Kubeconfig",
)
# set new vips
if tf and kube_client is None:
cluster_info = client.cluster_get(cluster.id)
if not cluster_info.api_vip:
cluster_info.api_vip = helper_cluster.get_api_vip_from_cluster(client, cluster_info)
tf.set_new_vip(cluster_info.api_vip)
def run_install_flow(client,
cluster_id,
kubeconfig_path,
pull_secret,
tf=None):
log.info("Verifying cluster exists")
cluster = client.cluster_get(cluster_id)
client.update_cluster_install_config(
cluster_id, {"networking": {
"networkType": "OpenShiftSDN"
}})
log.info("Verifying pull secret")
verify_pull_secret(client=client, cluster=cluster, pull_secret=pull_secret)
log.info("Wait till cluster is ready")
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster_id,
statuses=[consts.ClusterStatus.READY, consts.ClusterStatus.INSTALLING],
)
cluster = client.cluster_get(cluster_id)
if cluster.status == consts.ClusterStatus.READY:
log.info("Install cluster %s", cluster_id)
_install_cluster(client=client, cluster=cluster)
else:
log.info(
"Cluster is already in installing status, skipping install command"
)
log.info("Download kubeconfig-noingress")
client.download_kubeconfig_no_ingress(cluster_id=cluster_id,
kubeconfig_path=kubeconfig_path)
wait_till_installed(client=client, cluster=cluster)
log.info("Download kubeconfig")
waiting.wait(
lambda: client.download_kubeconfig(
cluster_id=cluster_id, kubeconfig_path=kubeconfig_path) is None,
timeout_seconds=240,
sleep_seconds=20,
expected_exceptions=Exception,
waiting_for="Kubeconfig",
)
# set new vips
if tf:
cluster_info = client.cluster_get(cluster.id)
tf.set_new_vip(cluster_info.api_vip)
def _install_cluster(client, cluster):
cluster = client.install_cluster(cluster_id=cluster.id)
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster.id,
timeout=consts.START_CLUSTER_INSTALLATION_TIMEOUT,
statuses=[consts.ClusterStatus.INSTALLING],
)
utils.wait_till_all_hosts_are_in_status(
client=client,
cluster_id=cluster.id,
nodes_count=len(cluster.hosts),
statuses=[
consts.NodesStatus.INSTALLING,
consts.NodesStatus.INSTALLING_IN_PROGRESS,
],
interval=30,
)
def wait_for_pending_for_input_status(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.PENDING_FOR_INPUT])
def wait_for_cluster_to_be_in_installing_status(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.INSTALLING])
def wait_until_cluster_is_ready_for_install(cluster_id, api_client):
utils.wait_till_cluster_is_in_status(
client=api_client,
cluster_id=cluster_id,
statuses=[consts.ClusterStatus.READY])
def wait_for_ready_to_install(self):
utils.wait_till_cluster_is_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.READY])
