def wait_till_installed(client, cluster, timeout=60 * 60 * 2):
log.info("Waiting %s till cluster finished installation", timeout)
# TODO: Change host validation for only previous known hosts
try:
utils.wait_till_all_hosts_are_in_status(
client=client,
cluster_id=cluster.id,
nodes_count=len(cluster.hosts),
statuses=[consts.NodesStatus.INSTALLED],
timeout=timeout,
interval=60,
)
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster.id,
statuses=[consts.ClusterStatus.INSTALLED],
timeout=consts.CLUSTER_INSTALLATION_TIMEOUT
if cluster.high_availability_mode == "Full" else
consts.CLUSTER_INSTALLATION_TIMEOUT * 2,
)
finally:
output_folder = f'build/{cluster.id}'
utils.recreate_folder(output_folder)
download_logs_from_all_hosts(client=client,
cluster_id=cluster.id,
output_folder=output_folder)
def create_nodes_and_wait_till_registered(cluster_name, inventory_client,
cluster, image_path, storage_path,
master_count, nodes_details, tf,
machine_net):
nodes_count = master_count + nodes_details["worker_count"]
create_nodes(cluster_name=cluster_name,
image_path=image_path,
storage_path=storage_path,
master_count=master_count,
nodes_details=nodes_details,
tf=tf,
machine_net=machine_net)
# TODO: Check for only new nodes
if not inventory_client:
# We will wait for leases only if only nodes are created without connection to s
utils.wait_till_nodes_are_ready(
nodes_count=nodes_count,
network_name=nodes_details["libvirt_network_name"])
log.info("No inventory url, will not wait till nodes registration")
return
log.info("Wait till nodes will be registered")
# In case there is assisted service connection, registration to the cluster in the assisted service
# is checked, and not relying on libvirt leases. This overcomes bug in libvirt that does not report
# all DHCP leases.
utils.wait_till_all_hosts_are_in_status(
client=inventory_client,
cluster_id=cluster.id,
nodes_count=nodes_count,
statuses=[
consts.NodesStatus.INSUFFICIENT,
consts.NodesStatus.PENDING_FOR_INPUT,
])
def wait_until_hosts_are_disconnected(
self, nodes_count=env_variables['num_nodes']):
statuses = [consts.NodesStatus.DISCONNECTED]
utils.wait_till_all_hosts_are_in_status(client=self.api_client,
cluster_id=self.id,
nodes_count=nodes_count,
statuses=statuses)
def wait_till_installed(client, cluster, timeout=60 * 60 * 2):
# TODO: Change host validation for only previous known hosts
try:
utils.wait_till_all_hosts_are_in_status(
client=client,
cluster_id=cluster.id,
nodes_count=len(cluster.hosts),
statuses=[consts.NodesStatus.INSTALLED],
timeout=timeout,
interval=60,
)
utils.wait_till_all_operators_are_in_status(
client=client,
cluster_id=cluster.id,
operators_count=len(cluster.monitored_operators),
operator_types=[OperatorType.BUILTIN, OperatorType.OLM],
statuses=[consts.OperatorStatus.AVAILABLE, consts.OperatorStatus.FAILED],
timeout=consts.CLUSTER_INSTALLATION_TIMEOUT,
fall_on_error_status=False,
)
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster.id,
statuses=[consts.ClusterStatus.INSTALLED],
timeout=consts.CLUSTER_INSTALLATION_TIMEOUT if cluster.high_availability_mode == "Full"
else consts.CLUSTER_INSTALLATION_TIMEOUT * 2,
break_statuses=[consts.ClusterStatus.ERROR]
)
finally:
output_folder = f'build/{cluster.id}'
utils.recreate_folder(output_folder)
download_logs_from_all_hosts(client=client, cluster_id=cluster.id, output_folder=output_folder)
def wait_until_hosts_are_disconnected(self, nodes_count: int = None):
statuses = [consts.NodesStatus.DISCONNECTED]
utils.wait_till_all_hosts_are_in_status(
client=self.api_client,
cluster_id=self.id,
nodes_count=nodes_count or self._config.nodes_count,
statuses=statuses,
timeout=consts.DISCONNECTED_TIMEOUT,
)
def wait_until_hosts_are_discovered(self,
nodes_count=env_variables['num_nodes']
):
utils.wait_till_all_hosts_are_in_status(
client=self.api_client,
cluster_id=self.id,
nodes_count=nodes_count,
statuses=[
consts.NodesStatus.PENDING_FOR_INPUT, consts.NodesStatus.KNOWN
])
def wait_for_hosts_to_install(self,
nodes_count=env_variables['num_nodes'],
timeout=consts.CLUSTER_INSTALLATION_TIMEOUT):
utils.wait_till_all_hosts_are_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.INSTALLED],
nodes_count=nodes_count,
timeout=timeout,
)
def wait_for_hosts_to_be_in_wrong_boot_order(
self, nodes_count, timeout=consts.PENDING_USER_ACTION_TIMEOUT, fall_on_error_status=True
):
utils.wait_till_all_hosts_are_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.NodesStatus.INSTALLING_PENDING_USER_ACTION],
nodes_count=nodes_count,
timeout=timeout,
fall_on_error_status=fall_on_error_status,
)
def wait_until_hosts_are_discovered(self, allow_insufficient=False, nodes_count: int = None):
statuses = [consts.NodesStatus.PENDING_FOR_INPUT, consts.NodesStatus.KNOWN]
if allow_insufficient:
statuses.append(consts.NodesStatus.INSUFFICIENT)
utils.wait_till_all_hosts_are_in_status(
client=self.api_client,
cluster_id=self.id,
nodes_count=nodes_count or self._config.nodes_count,
statuses=statuses,
timeout=consts.NODES_REGISTERED_TIMEOUT,
)
def wait_for_hosts_to_install(
self, timeout=consts.CLUSTER_INSTALLATION_TIMEOUT, fall_on_error_status=True, nodes_count: int = None
):
utils.wait_till_all_hosts_are_in_status(
client=self.api_client,
cluster_id=self.id,
statuses=[consts.ClusterStatus.INSTALLED],
nodes_count=nodes_count or self._config.nodes_count,
timeout=timeout,
fall_on_error_status=fall_on_error_status,
)
def wait_until_hosts_are_discovered(self,
nodes_count=env_variables['num_nodes'],
allow_insufficient=False):
statuses = [
consts.NodesStatus.PENDING_FOR_INPUT, consts.NodesStatus.KNOWN
]
if allow_insufficient:
statuses.append(consts.NodesStatus.INSUFFICIENT)
utils.wait_till_all_hosts_are_in_status(client=self.api_client,
cluster_id=self.id,
nodes_count=nodes_count,
statuses=statuses)
def wait_until_hosts_are_registered(
cluster_id,
api_client,
nodes_count=env_variables['num_masters'] +
env_variables['num_workers']):
utils.wait_till_all_hosts_are_in_status(
client=api_client,
cluster_id=cluster_id,
nodes_count=nodes_count,
statuses=[
consts.NodesStatus.PENDING_FOR_INPUT,
consts.NodesStatus.INSUFFICIENT,
consts.NodesStatus.KNOWN,
])
def wait_for_nodes_status_installing_or_installed(
cluster_id,
api_client,
nodes_count=env_variables['num_masters'] +
env_variables['num_workers']):
utils.wait_till_all_hosts_are_in_status(
client=api_client,
cluster_id=cluster_id,
nodes_count=nodes_count,
statuses=[
consts.NodesStatus.INSTALLING_IN_PROGRESS,
consts.NodesStatus.INSTALLED
],
interval=30,
)
def _install_cluster(client, cluster):
cluster = client.install_cluster(cluster_id=cluster.id)
utils.wait_till_cluster_is_in_status(
client=client,
cluster_id=cluster.id,
timeout=consts.START_CLUSTER_INSTALLATION_TIMEOUT,
statuses=[consts.ClusterStatus.INSTALLING],
)
utils.wait_till_all_hosts_are_in_status(
client=client,
cluster_id=cluster.id,
nodes_count=len(cluster.hosts),
statuses=[
consts.NodesStatus.INSTALLING,
consts.NodesStatus.INSTALLING_IN_PROGRESS,
],
interval=30,
)
def set_network_params(
cluster_id,
api_client,
controller,
nodes_count=env_variables['num_masters'] +
env_variables['num_workers'],
vip_dhcp_allocation=env_variables['vip_dhcp_allocation'],
cluster_machine_cidr=env_variables['machine_cidr']):
if vip_dhcp_allocation:
BaseTest.set_cluster_machine_cidr(cluster_id, api_client,
cluster_machine_cidr)
else:
BaseTest.set_ingress_and_api_vips(cluster_id, api_client,
controller)
utils.wait_till_all_hosts_are_in_status(
client=api_client,
cluster_id=cluster_id,
nodes_count=nodes_count,
statuses=[consts.NodesStatus.KNOWN])
def wait_until_nodes_are_registered_rest_api(
inventory_client,
cluster,
nodes_details,
is_ipv4,
nodes_number,
):
# TODO: Check for only new nodes
if not inventory_client:
# We will wait for leases only if only nodes are created without connection to s
utils.wait_till_nodes_are_ready(
nodes_count=nodes_number,
network_name=nodes_details["libvirt_network_name"],
)
log.info("No inventory url, will not wait till nodes registration")
return
log.info("Wait till nodes will be registered")
# In case there is assisted service connection, registration to the cluster in the assisted service
# is checked, and not relying on libvirt leases. This overcomes bug in libvirt that does not report
# all DHCP leases.
statuses = [
consts.NodesStatus.INSUFFICIENT,
consts.NodesStatus.PENDING_FOR_INPUT,
]
if nodes_details['master_count'] == 1 or is_none_platform_mode():
statuses.append(consts.NodesStatus.KNOWN)
if is_ipv4 and is_none_platform_mode(
) and nodes_details['master_count'] > 1:
input_interfaces = [args.network_bridge, f"s{args.network_bridge}"]
nat_controller = NatController()
nat_controller.add_nat_rules(input_interfaces, args.ns_index)
utils.wait_till_all_hosts_are_in_status(
client=inventory_client,
cluster_id=cluster.id,
nodes_count=nodes_number,
statuses=statuses,
)
def day2_nodes_flow(client, terraform_cluster_dir_prefix, tf_folder, cluster,
has_ipv_6, num_worker_nodes, api_vip_ip, api_vip_dnsname,
install_cluster_flag, day2_type_flag,
with_static_network_config, base_cluster_name):
tf_network_name, total_num_nodes = get_network_num_nodes_from_tf(tf_folder)
with utils.file_lock_context():
utils.run_command(
f'make _apply_terraform CLUSTER_NAME={terraform_cluster_dir_prefix}'
)
time.sleep(5)
if day2_type_flag == "ocp":
num_nodes_to_wait = total_num_nodes
installed_status = consts.NodesStatus.INSTALLED
else:
num_nodes_to_wait = num_worker_nodes
installed_status = consts.NodesStatus.DAY2_INSTALLED
utils.wait_till_nodes_are_ready(nodes_count=num_nodes_to_wait,
network_name=tf_network_name)
waiting.wait(
lambda: utils.are_libvirt_nodes_in_cluster_hosts(
client, cluster.id, num_nodes_to_wait),
timeout_seconds=consts.NODES_REGISTERED_TIMEOUT,
sleep_seconds=10,
waiting_for="Nodes to be registered in inventory service",
)
set_nodes_hostnames_if_needed(client, tf_folder,
with_static_network_config, has_ipv_6,
tf_network_name, cluster.id)
utils.wait_till_all_hosts_are_in_status(
client=client,
cluster_id=cluster.id,
nodes_count=num_worker_nodes,
statuses=[consts.NodesStatus.KNOWN],
interval=30,
)
if install_cluster_flag:
log.info("Start installing all known nodes in the cluster %s",
cluster.id)
kubeconfig = utils.get_kubeconfig_path(base_cluster_name)
ocp_orig_ready_nodes = get_ocp_cluster_ready_nodes_num(kubeconfig)
hosts = client.get_cluster_hosts(cluster.id)
[
client.install_day2_host(cluster.id, host['id']) for host in hosts
if host["status"] == 'known'
]
log.info(
"Start waiting until all nodes of cluster %s have been installed( reached added-to-existing-clustertate)",
cluster.id)
utils.wait_till_all_hosts_are_in_status(
client=client,
cluster_id=cluster.id,
nodes_count=num_nodes_to_wait,
statuses=[installed_status],
interval=30,
)
log.info(
"Start waiting until installed nodes has actually been added to the OCP cluster"
)
waiting.wait(lambda: wait_nodes_join_ocp_cluster(
ocp_orig_ready_nodes, num_worker_nodes, day2_type_flag, kubeconfig
),
timeout_seconds=consts.NODES_REGISTERED_TIMEOUT,
sleep_seconds=30,
waiting_for="Day2 nodes to be added to OCP cluster",
expected_exceptions=Exception)
log.info("%d worker nodes were successfully added to OCP cluster",
num_worker_nodes)
