def run_ocs_upgrade(operation=None, *operation_args, **operation_kwargs):
"""
Run upgrade procedure of OCS cluster
Args:
operation: (function): Function to run
operation_args: (iterable): Function's arguments
operation_kwargs: (map): Function's keyword arguments
"""
ceph_cluster = CephCluster()
upgrade_ocs = OCSUpgrade(
namespace=config.ENV_DATA['cluster_namespace'],
version_before_upgrade=config.ENV_DATA.get("ocs_version"),
ocs_registry_image=config.UPGRADE.get('upgrade_ocs_registry_image'),
upgrade_in_current_source=config.UPGRADE.get('upgrade_in_current_source', False)
)
upgrade_version = upgrade_ocs.get_upgrade_version()
assert upgrade_ocs.get_parsed_versions()[1] >= upgrade_ocs.get_parsed_versions()[0], (
f"Version you would like to upgrade to: {upgrade_version} "
f"is not higher or equal to the version you currently running: "
f"{upgrade_ocs.version_before_upgrade}"
)
csv_name_pre_upgrade = upgrade_ocs.get_csv_name_pre_upgrade()
pre_upgrade_images = upgrade_ocs.get_pre_upgrade_image(csv_name_pre_upgrade)
upgrade_ocs.load_version_config_file(upgrade_version)
with CephHealthMonitor(ceph_cluster):
channel = upgrade_ocs.set_upgrade_channel()
upgrade_ocs.set_upgrade_images()
upgrade_ocs.update_subscription(channel)
if operation:
log.info(f"Calling test function: {operation}")
_ = operation(*operation_args, **operation_kwargs)
# Workaround for issue #2531
time.sleep(30)
# End of workaround
for sample in TimeoutSampler(
timeout=725,
sleep=5,
func=upgrade_ocs.check_if_upgrade_completed,
channel=channel,
csv_name_pre_upgrade=csv_name_pre_upgrade,
):
try:
if sample:
log.info("Upgrade success!")
break
except TimeoutException:
raise TimeoutException("No new CSV found after upgrade!")
old_image = upgrade_ocs.get_images_post_upgrade(
channel, pre_upgrade_images, upgrade_version
)
verify_image_versions(old_image, upgrade_ocs.get_parsed_versions()[1])
ocs_install_verification(
timeout=600, skip_osd_distribution_check=True,
ocs_registry_image=upgrade_ocs.ocs_registry_image,
post_upgrade_verification=True,
)
def test_upgrade_ocp(self):
"""
Tests OCS stability when upgrading OCP
"""
ceph_cluster = CephCluster()
with CephHealthMonitor(ceph_cluster):
ocp_upgrade_version = config.UPGRADE.get('ocp_upgrade_version')
if not ocp_upgrade_version:
ocp_channel = config.UPGRADE['ocp_channel']
ocp_upgrade_version = get_latest_ocp_version(
channel=ocp_channel)
ocp_arch = config.UPGRADE['ocp_arch']
target_image = f"{ocp_upgrade_version}-{ocp_arch}"
elif ocp_upgrade_version.endswith(".nightly"):
target_image = expose_ocp_version(ocp_upgrade_version)
logger.info(f"Target image; {target_image}")
image_path = config.UPGRADE['ocp_upgrade_path']
self.cluster_operators = ocp.get_all_cluster_operators()
logger.info(f" oc version: {ocp.get_current_oc_version()}")
# Upgrade OCP
logger.info(f"full upgrade path: {image_path}:{target_image}")
ocp.upgrade_ocp(image=target_image, image_path=image_path)
# Wait for upgrade
for ocp_operator in self.cluster_operators:
logger.info(f"Checking upgrade status of {ocp_operator}:")
ver = ocp.get_cluster_operator_version(ocp_operator)
logger.info(f"current {ocp_operator} version: {ver}")
for sampler in TimeoutSampler(
timeout=2700,
sleep=60,
func=ocp.confirm_cluster_operator_version,
target_version=target_image,
cluster_operator=ocp_operator):
logger.info(
f"ClusterOperator upgrade "
f"{'completed!' if sampler else 'did not completed yet!'}"
)
if sampler:
break
# post upgrade validation: check cluster operator status
for ocp_operator in self.cluster_operators:
logger.info(f"Checking cluster status of {ocp_operator}")
for sampler in TimeoutSampler(
timeout=2700,
sleep=60,
func=ocp.verify_cluster_operator_status,
cluster_operator=ocp_operator):
logger.info(
f"ClusterOperator status is "
f"{'valid' if sampler else 'status is not valid'}")
if sampler:
break
def test_upgrade_logging(self):
"""
This function contains test to upgrade openshift-logging
with Entry and Exit criteria for checks
"""
ceph_cluster = CephCluster()
with CephHealthMonitor(ceph_cluster):
# Pre-check
logger.info("Checking cluster logging before starting to upgrade")
check_cluster_logging()
# Matching the OCP version and cluster-Logging version
ocp_version = get_ocp_version()
logger.info(f"OCP version {ocp_version}")
subscription = get_clusterlogging_subscription()
logging_channel = subscription.get('spec').get('channel')
logger.info(
f"Current Logging channel {logging_channel}"
)
upgrade_info(logging_channel)
upgrade_channel = config.UPGRADE['upgrade_logging_channel']
if ocp_version > logging_channel:
# Upgrade Elastic search operator Subscription
es_subscription_cmd = (
'oc patch subscription elasticsearch-operator '
f'-n {constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE} '
'--type merge -p \'{"spec":{"channel": '
f'"{upgrade_channel}"}}}}\''
)
# Upgrade Cluster-logging operator subscription
clo_subscription_cmd = (
'oc patch subscription cluster-logging '
f'-n {constants.OPENSHIFT_LOGGING_NAMESPACE} '
'--type merge -p \'{"spec":{"channel": '
f'"{upgrade_channel}"}}}}\''
)
run_cmd(es_subscription_cmd)
run_cmd(clo_subscription_cmd)
assert check_csv_logging_phase(
upgrade_channel
), "Logging upgrade not completed yet!"
logger.info(
"Logging upgrade completed!"
)
assert check_csv_version_post_upgrade(
upgrade_channel
), "Unable to get version "
logger.info("Version Matches!")
upgrade_info(upgrade_channel)
check_cluster_logging()
else:
logger.info(
"Logging Version matches the OCP version, "
"No upgrade needed"
)
def test_upgrade():
"""
Tests upgrade procedure of OCS cluster
"""
ceph_cluster = CephCluster()
upgrade_ocs = OCSUpgrade(
namespace=config.ENV_DATA['cluster_namespace'],
version_before_upgrade=config.ENV_DATA.get("ocs_version"),
ocs_registry_image=config.UPGRADE.get('upgrade_ocs_registry_image'),
upgrade_in_current_source=config.UPGRADE.get('upgrade_in_current_source', False)
)
upgrade_version = upgrade_ocs.get_upgrade_version()
assert upgrade_ocs.get_parsed_versions()[1] >= upgrade_ocs.get_parsed_versions()[0], (
f"Version you would like to upgrade to: {upgrade_version} "
f"is not higher or equal to the version you currently running: "
f"{upgrade_ocs.version_before_upgrade}"
)
csv_name_pre_upgrade = upgrade_ocs.get_csv_name_pre_upgrade()
pre_upgrade_images = upgrade_ocs.get_pre_upgrade_image(csv_name_pre_upgrade)
upgrade_ocs.load_version_config_file(upgrade_version)
with CephHealthMonitor(ceph_cluster):
channel = upgrade_ocs.set_upgrade_channel()
upgrade_ocs.set_upgrade_images()
upgrade_ocs.update_subscription(channel)
for sample in TimeoutSampler(
timeout=725,
sleep=5,
func=upgrade_ocs.check_if_upgrade_completed,
channel=channel,
csv_name_pre_upgrade=csv_name_pre_upgrade,
):
try:
if sample:
log.info("Upgrade success!")
break
except TimeoutException:
raise TimeoutException("No new CSV found after upgrade!")
old_image = upgrade_ocs.get_images_post_upgrade(
channel, pre_upgrade_images, upgrade_version
)
verify_image_versions(old_image, upgrade_ocs.get_parsed_versions()[1])
ocs_install_verification(
timeout=600, skip_osd_distribution_check=True,
ocs_registry_image=upgrade_ocs.ocs_registry_image,
post_upgrade_verification=True,
)
def run_ocs_upgrade(operation=None, *operation_args, **operation_kwargs):
"""
Run upgrade procedure of OCS cluster
Args:
operation: (function): Function to run
operation_args: (iterable): Function's arguments
operation_kwargs: (map): Function's keyword arguments
"""
ceph_cluster = CephCluster()
original_ocs_version = config.ENV_DATA.get("ocs_version")
upgrade_ocs = OCSUpgrade(
namespace=config.ENV_DATA["cluster_namespace"],
version_before_upgrade=original_ocs_version,
ocs_registry_image=config.UPGRADE.get("upgrade_ocs_registry_image"),
upgrade_in_current_source=config.UPGRADE.get(
"upgrade_in_current_source", False),
)
upgrade_version = upgrade_ocs.get_upgrade_version()
assert (
upgrade_ocs.get_parsed_versions()[1] >=
upgrade_ocs.get_parsed_versions()[0]), (
f"Version you would like to upgrade to: {upgrade_version} "
f"is not higher or equal to the version you currently running: "
f"{upgrade_ocs.version_before_upgrade}")
# For external cluster , create the secrets if upgraded version is 4.8
if (config.DEPLOYMENT["external_mode"] and original_ocs_version == "4.7"
and upgrade_version == "4.8"):
access_key = config.EXTERNAL_MODE.get("access_key_rgw-admin-ops-user",
"")
secret_key = config.EXTERNAL_MODE.get("secret_key_rgw-admin-ops-user",
"")
if not (access_key and secret_key):
raise ExternalClusterRGWAdminOpsUserException(
"Access and secret key for rgw-admin-ops-user not found")
cmd = (
f'oc create secret generic --type="kubernetes.io/rook"'
f' "rgw-admin-ops-user" --from-literal=accessKey={access_key} --from-literal=secretKey={secret_key}'
)
exec_cmd(cmd)
csv_name_pre_upgrade = upgrade_ocs.get_csv_name_pre_upgrade()
pre_upgrade_images = upgrade_ocs.get_pre_upgrade_image(
csv_name_pre_upgrade)
upgrade_ocs.load_version_config_file(upgrade_version)
if config.DEPLOYMENT.get("disconnected"):
upgrade_ocs.ocs_registry_image = prepare_disconnected_ocs_deployment(
upgrade=True)
log.info(
f"Disconnected upgrade - new image: {upgrade_ocs.ocs_registry_image}"
)
with CephHealthMonitor(ceph_cluster):
channel = upgrade_ocs.set_upgrade_channel()
upgrade_ocs.set_upgrade_images()
upgrade_ocs.update_subscription(channel)
if operation:
log.info(f"Calling test function: {operation}")
_ = operation(*operation_args, **operation_kwargs)
# Workaround for issue #2531
time.sleep(30)
# End of workaround
for sample in TimeoutSampler(
timeout=725,
sleep=5,
func=upgrade_ocs.check_if_upgrade_completed,
channel=channel,
csv_name_pre_upgrade=csv_name_pre_upgrade,
):
try:
if sample:
log.info("Upgrade success!")
break
except TimeoutException:
raise TimeoutException("No new CSV found after upgrade!")
old_image = upgrade_ocs.get_images_post_upgrade(
channel, pre_upgrade_images, upgrade_version)
verify_image_versions(
old_image,
upgrade_ocs.get_parsed_versions()[1],
upgrade_ocs.version_before_upgrade,
)
ocs_install_verification(
timeout=600,
skip_osd_distribution_check=True,
ocs_registry_image=upgrade_ocs.ocs_registry_image,
post_upgrade_verification=True,
version_before_upgrade=upgrade_ocs.version_before_upgrade,
)
def test_upgrade_ocp(self, reduce_and_resume_cluster_load):
"""
Tests OCS stability when upgrading OCP
"""
ceph_cluster = CephCluster()
with CephHealthMonitor(ceph_cluster):
ocp_channel = config.UPGRADE.get("ocp_channel",
ocp.get_ocp_upgrade_channel())
ocp_upgrade_version = config.UPGRADE.get("ocp_upgrade_version")
if not ocp_upgrade_version:
ocp_upgrade_version = get_latest_ocp_version(
channel=ocp_channel)
ocp_arch = config.UPGRADE["ocp_arch"]
target_image = f"{ocp_upgrade_version}-{ocp_arch}"
elif ocp_upgrade_version.endswith(".nightly"):
target_image = expose_ocp_version(ocp_upgrade_version)
logger.info(f"Target image; {target_image}")
image_path = config.UPGRADE["ocp_upgrade_path"]
cluster_operators = ocp.get_all_cluster_operators()
logger.info(f" oc version: {ocp.get_current_oc_version()}")
# Verify Upgrade subscription channel:
ocp.patch_ocp_upgrade_channel(ocp_channel)
for sampler in TimeoutSampler(
timeout=250,
sleep=15,
func=ocp.verify_ocp_upgrade_channel,
channel_variable=ocp_channel,
):
if sampler:
logger.info(f"OCP Channel:{ocp_channel}")
break
# Upgrade OCP
logger.info(f"full upgrade path: {image_path}:{target_image}")
ocp.upgrade_ocp(image=target_image, image_path=image_path)
# Wait for upgrade
for ocp_operator in cluster_operators:
logger.info(f"Checking upgrade status of {ocp_operator}:")
# ############ Workaround for issue 2624 #######
name_changed_between_versions = (
"service-catalog-apiserver",
"service-catalog-controller-manager",
)
if ocp_operator in name_changed_between_versions:
logger.info(f"{ocp_operator} upgrade will not be verified")
continue
# ############ End of Workaround ###############
ver = ocp.get_cluster_operator_version(ocp_operator)
logger.info(f"current {ocp_operator} version: {ver}")
for sampler in TimeoutSampler(
timeout=2700,
sleep=60,
func=ocp.confirm_cluster_operator_version,
target_version=target_image,
cluster_operator=ocp_operator,
):
if sampler:
logger.info(f"{ocp_operator} upgrade completed!")
break
else:
logger.info(
f"{ocp_operator} upgrade did not completed yet!")
# post upgrade validation: check cluster operator status
cluster_operators = ocp.get_all_cluster_operators()
for ocp_operator in cluster_operators:
logger.info(f"Checking cluster status of {ocp_operator}")
for sampler in TimeoutSampler(
timeout=2700,
sleep=60,
func=ocp.verify_cluster_operator_status,
cluster_operator=ocp_operator,
):
if sampler:
break
else:
logger.info(f"{ocp_operator} status is not valid")
# Post upgrade validation: check cluster version status
logger.info("Checking clusterversion status")
for sampler in TimeoutSampler(
timeout=900,
sleep=15,
func=ocp.validate_cluster_version_status):
if sampler:
logger.info("Upgrade Completed Successfully!")
break
new_ceph_cluster = CephCluster()
new_ceph_cluster.wait_for_rebalance(timeout=1800)
ceph_health_check(tries=90, delay=30)
def run_ocs_upgrade(operation=None, *operation_args, **operation_kwargs):
"""
Run upgrade procedure of OCS cluster
Args:
operation: (function): Function to run
operation_args: (iterable): Function's arguments
operation_kwargs: (map): Function's keyword arguments
"""
ceph_cluster = CephCluster()
original_ocs_version = config.ENV_DATA.get("ocs_version")
upgrade_in_current_source = config.UPGRADE.get("upgrade_in_current_source", False)
upgrade_ocs = OCSUpgrade(
namespace=config.ENV_DATA["cluster_namespace"],
version_before_upgrade=original_ocs_version,
ocs_registry_image=config.UPGRADE.get("upgrade_ocs_registry_image"),
upgrade_in_current_source=upgrade_in_current_source,
)
upgrade_version = upgrade_ocs.get_upgrade_version()
assert (
upgrade_ocs.get_parsed_versions()[1] >= upgrade_ocs.get_parsed_versions()[0]
), (
f"Version you would like to upgrade to: {upgrade_version} "
f"is not higher or equal to the version you currently running: "
f"{upgrade_ocs.version_before_upgrade}"
)
# For external cluster , create the secrets if upgraded version is 4.8
if (
config.DEPLOYMENT["external_mode"]
and original_ocs_version == "4.7"
and upgrade_version == "4.8"
):
access_key = config.EXTERNAL_MODE.get("access_key_rgw-admin-ops-user", "")
secret_key = config.EXTERNAL_MODE.get("secret_key_rgw-admin-ops-user", "")
if not (access_key and secret_key):
raise ExternalClusterRGWAdminOpsUserException(
"Access and secret key for rgw-admin-ops-user not found"
)
cmd = (
f'oc create secret generic --type="kubernetes.io/rook"'
f' "rgw-admin-ops-user" --from-literal=accessKey={access_key} --from-literal=secretKey={secret_key}'
)
exec_cmd(cmd)
csv_name_pre_upgrade = upgrade_ocs.get_csv_name_pre_upgrade()
pre_upgrade_images = upgrade_ocs.get_pre_upgrade_image(csv_name_pre_upgrade)
upgrade_ocs.load_version_config_file(upgrade_version)
if config.DEPLOYMENT.get("disconnected"):
upgrade_ocs.ocs_registry_image = prepare_disconnected_ocs_deployment(
upgrade=True
)
log.info(f"Disconnected upgrade - new image: {upgrade_ocs.ocs_registry_image}")
with CephHealthMonitor(ceph_cluster):
channel = upgrade_ocs.set_upgrade_channel()
upgrade_ocs.set_upgrade_images()
ui_upgrade_supported = False
if config.UPGRADE.get("ui_upgrade"):
if (
version.get_semantic_ocp_version_from_config() == version.VERSION_4_9
and original_ocs_version == "4.8"
and upgrade_version == "4.9"
):
ui_upgrade_supported = True
else:
log.warning(
"UI upgrade combination is not supported. It will fallback to CLI upgrade"
)
if ui_upgrade_supported:
ocs_odf_upgrade_ui()
else:
if upgrade_version != "4.9":
# In the case of upgrade to ODF 4.9, the ODF operator should upgrade
# OCS automatically.
upgrade_ocs.update_subscription(channel)
if original_ocs_version == "4.8" and upgrade_version == "4.9":
deployment = Deployment()
deployment.subscribe_ocs()
else:
# In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy
# we need to wait and approve install plan, otherwise it's approved in the
# subscribe_ocs method.
subscription_plan_approval = config.DEPLOYMENT.get(
"subscription_plan_approval"
)
if subscription_plan_approval == "Manual":
wait_for_install_plan_and_approve(
config.ENV_DATA["cluster_namespace"]
)
if (config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM) and not (
upgrade_in_current_source
):
create_ocs_secret(config.ENV_DATA["cluster_namespace"])
for attempt in range(2):
# We need to do it twice, because some of the SA are updated
# after the first load of OCS pod after upgrade. So we need to
# link updated SA again.
log.info(
f"Sleep 1 minute before attempt: {attempt + 1}/2 "
"of linking secret/SAs"
)
time.sleep(60)
link_all_sa_and_secret_and_delete_pods(
constants.OCS_SECRET, config.ENV_DATA["cluster_namespace"]
)
if operation:
log.info(f"Calling test function: {operation}")
_ = operation(*operation_args, **operation_kwargs)
# Workaround for issue #2531
time.sleep(30)
# End of workaround
for sample in TimeoutSampler(
timeout=725,
sleep=5,
func=upgrade_ocs.check_if_upgrade_completed,
channel=channel,
csv_name_pre_upgrade=csv_name_pre_upgrade,
):
try:
if sample:
log.info("Upgrade success!")
break
except TimeoutException:
raise TimeoutException("No new CSV found after upgrade!")
old_image = upgrade_ocs.get_images_post_upgrade(
channel, pre_upgrade_images, upgrade_version
)
verify_image_versions(
old_image,
upgrade_ocs.get_parsed_versions()[1],
upgrade_ocs.version_before_upgrade,
)
ocs_install_verification(
timeout=600,
skip_osd_distribution_check=True,
ocs_registry_image=upgrade_ocs.ocs_registry_image,
post_upgrade_verification=True,
version_before_upgrade=upgrade_ocs.version_before_upgrade,
)
def run_ocs_upgrade(operation=None, *operation_args, **operation_kwargs):
"""
Run upgrade procedure of OCS cluster
Args:
operation: (function): Function to run
operation_args: (iterable): Function's arguments
operation_kwargs: (map): Function's keyword arguments
"""
ceph_cluster = CephCluster()
original_ocs_version = config.ENV_DATA.get("ocs_version")
upgrade_in_current_source = config.UPGRADE.get("upgrade_in_current_source",
False)
upgrade_ocs = OCSUpgrade(
namespace=config.ENV_DATA["cluster_namespace"],
version_before_upgrade=original_ocs_version,
ocs_registry_image=config.UPGRADE.get("upgrade_ocs_registry_image"),
upgrade_in_current_source=upgrade_in_current_source,
)
upgrade_version = upgrade_ocs.get_upgrade_version()
assert (
upgrade_ocs.get_parsed_versions()[1] >=
upgrade_ocs.get_parsed_versions()[0]), (
f"Version you would like to upgrade to: {upgrade_version} "
f"is not higher or equal to the version you currently running: "
f"{upgrade_ocs.version_before_upgrade}")
# create external cluster object
if config.DEPLOYMENT["external_mode"]:
host, user, password = get_external_cluster_client()
external_cluster = ExternalCluster(host, user, password)
# For external cluster , create the secrets if upgraded version is 4.8
if (config.DEPLOYMENT["external_mode"] and original_ocs_version == "4.7"
and upgrade_version == "4.8"):
external_cluster.create_object_store_user()
access_key = config.EXTERNAL_MODE.get("access_key_rgw-admin-ops-user",
"")
secret_key = config.EXTERNAL_MODE.get("secret_key_rgw-admin-ops-user",
"")
if not (access_key and secret_key):
raise ExternalClusterRGWAdminOpsUserException(
"Access and secret key for rgw-admin-ops-user not found")
cmd = (
f'oc create secret generic --type="kubernetes.io/rook"'
f' "rgw-admin-ops-user" --from-literal=accessKey={access_key} --from-literal=secretKey={secret_key}'
)
exec_cmd(cmd)
csv_name_pre_upgrade = upgrade_ocs.get_csv_name_pre_upgrade()
pre_upgrade_images = upgrade_ocs.get_pre_upgrade_image(
csv_name_pre_upgrade)
upgrade_ocs.load_version_config_file(upgrade_version)
if config.DEPLOYMENT.get("disconnected") and not config.DEPLOYMENT.get(
"disconnected_env_skip_image_mirroring"):
upgrade_ocs.ocs_registry_image = prepare_disconnected_ocs_deployment(
upgrade=True)
log.info(
f"Disconnected upgrade - new image: {upgrade_ocs.ocs_registry_image}"
)
with CephHealthMonitor(ceph_cluster):
channel = upgrade_ocs.set_upgrade_channel()
upgrade_ocs.set_upgrade_images()
live_deployment = config.DEPLOYMENT["live_deployment"]
disable_addon = config.DEPLOYMENT.get("ibmcloud_disable_addon")
if (config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM
and live_deployment and not disable_addon):
clustername = config.ENV_DATA.get("cluster_name")
cmd = f"ibmcloud ks cluster addon disable openshift-data-foundation --cluster {clustername} -f"
run_ibmcloud_cmd(cmd)
time.sleep(120)
cmd = (
f"ibmcloud ks cluster addon enable openshift-data-foundation --cluster {clustername} -f --version "
f"{upgrade_version}.0 --param ocsUpgrade=true")
run_ibmcloud_cmd(cmd)
time.sleep(120)
else:
ui_upgrade_supported = False
if config.UPGRADE.get("ui_upgrade"):
if (version.get_semantic_ocp_version_from_config()
== version.VERSION_4_9
and original_ocs_version == "4.8"
and upgrade_version == "4.9"):
ui_upgrade_supported = True
else:
log.warning(
"UI upgrade combination is not supported. It will fallback to CLI upgrade"
)
if ui_upgrade_supported:
ocs_odf_upgrade_ui()
else:
if (config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM
) and not (upgrade_in_current_source):
create_ocs_secret(config.ENV_DATA["cluster_namespace"])
if upgrade_version != "4.9":
# In the case of upgrade to ODF 4.9, the ODF operator should upgrade
# OCS automatically.
upgrade_ocs.update_subscription(channel)
if original_ocs_version == "4.8" and upgrade_version == "4.9":
deployment = Deployment()
deployment.subscribe_ocs()
else:
# In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy
# we need to wait and approve install plan, otherwise it's approved in the
# subscribe_ocs method.
subscription_plan_approval = config.DEPLOYMENT.get(
"subscription_plan_approval")
if subscription_plan_approval == "Manual":
wait_for_install_plan_and_approve(
config.ENV_DATA["cluster_namespace"])
if (config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM
) and not (upgrade_in_current_source):
for attempt in range(2):
# We need to do it twice, because some of the SA are updated
# after the first load of OCS pod after upgrade. So we need to
# link updated SA again.
log.info(
f"Sleep 1 minute before attempt: {attempt + 1}/2 "
"of linking secret/SAs")
time.sleep(60)
link_all_sa_and_secret_and_delete_pods(
constants.OCS_SECRET,
config.ENV_DATA["cluster_namespace"])
if operation:
log.info(f"Calling test function: {operation}")
_ = operation(*operation_args, **operation_kwargs)
# Workaround for issue #2531
time.sleep(30)
# End of workaround
for sample in TimeoutSampler(
timeout=725,
sleep=5,
func=upgrade_ocs.check_if_upgrade_completed,
channel=channel,
csv_name_pre_upgrade=csv_name_pre_upgrade,
):
try:
if sample:
log.info("Upgrade success!")
break
except TimeoutException:
raise TimeoutException("No new CSV found after upgrade!")
old_image = upgrade_ocs.get_images_post_upgrade(
channel, pre_upgrade_images, upgrade_version)
verify_image_versions(
old_image,
upgrade_ocs.get_parsed_versions()[1],
upgrade_ocs.version_before_upgrade,
)
# update external secrets
if config.DEPLOYMENT["external_mode"]:
upgrade_version = version.get_semantic_version(upgrade_version, True)
if upgrade_version >= version.VERSION_4_10:
external_cluster.update_permission_caps()
else:
external_cluster.update_permission_caps(EXTERNAL_CLUSTER_USER)
external_cluster.get_external_cluster_details()
# update the external cluster details in secrets
log.info("updating external cluster secret")
external_cluster_details = NamedTemporaryFile(
mode="w+",
prefix="external-cluster-details-",
delete=False,
)
with open(external_cluster_details.name, "w") as fd:
decoded_external_cluster_details = decode(
config.EXTERNAL_MODE["external_cluster_details"])
fd.write(decoded_external_cluster_details)
cmd = (
f"oc set data secret/rook-ceph-external-cluster-details -n {constants.OPENSHIFT_STORAGE_NAMESPACE} "
f"--from-file=external_cluster_details={external_cluster_details.name}"
)
exec_cmd(cmd)
if config.ENV_DATA.get("mcg_only_deployment"):
mcg_only_install_verification(
ocs_registry_image=upgrade_ocs.ocs_registry_image)
else:
ocs_install_verification(
timeout=600,
skip_osd_distribution_check=True,
ocs_registry_image=upgrade_ocs.ocs_registry_image,
post_upgrade_verification=True,
version_before_upgrade=upgrade_ocs.version_before_upgrade,
)
def test_upgrade():
ceph_cluster = CephCluster()
with CephHealthMonitor(ceph_cluster):
namespace = config.ENV_DATA['cluster_namespace']
version_before_upgrade = config.ENV_DATA.get("ocs_version")
upgrade_version = config.UPGRADE.get("upgrade_ocs_version",
version_before_upgrade)
ocs_registry_image = config.UPGRADE.get('upgrade_ocs_registry_image')
if ocs_registry_image:
upgrade_version = get_ocs_version_from_image(ocs_registry_image)
parsed_version_before_upgrade = parse_version(version_before_upgrade)
parsed_upgrade_version = parse_version(upgrade_version)
assert parsed_upgrade_version >= parsed_version_before_upgrade, (
f"Version you would like to upgrade to: {upgrade_version} "
f"is not higher or equal to the version you currently running: "
f"{version_before_upgrade}")
operator_selector = get_selector_for_ocs_operator()
package_manifest = PackageManifest(
resource_name=OCS_OPERATOR_NAME,
selector=operator_selector,
)
channel = config.DEPLOYMENT.get('ocs_csv_channel')
csv_name_pre_upgrade = package_manifest.get_current_csv(channel)
log.info(f"CSV name before upgrade is: {csv_name_pre_upgrade}")
csv_pre_upgrade = CSV(resource_name=csv_name_pre_upgrade,
namespace=namespace)
pre_upgrade_images = get_images(csv_pre_upgrade.get())
version_change = parsed_upgrade_version > parsed_version_before_upgrade
if version_change:
version_config_file = os.path.join(constants.CONF_DIR,
'ocs_version',
f'ocs-{upgrade_version}.yaml')
load_config_file(version_config_file)
ocs_catalog = CatalogSource(
resource_name=constants.OPERATOR_CATALOG_SOURCE_NAME,
namespace=constants.MARKETPLACE_NAMESPACE,
)
upgrade_in_current_source = config.UPGRADE.get(
'upgrade_in_current_source', False)
if not upgrade_in_current_source:
if not ocs_catalog.is_exist() and not upgrade_in_current_source:
log.info("OCS catalog source doesn't exist. Creating new one.")
create_catalog_source(ocs_registry_image, ignore_upgrade=True)
image_url = ocs_catalog.get_image_url()
image_tag = ocs_catalog.get_image_name()
log.info(f"Current image is: {image_url}, tag: {image_tag}")
if ocs_registry_image:
image_url, new_image_tag = ocs_registry_image.split(':')
elif (config.UPGRADE.get('upgrade_to_latest', True)
or version_change):
new_image_tag = get_latest_ds_olm_tag()
else:
new_image_tag = get_next_version_available_for_upgrade(
image_tag)
cs_data = deepcopy(ocs_catalog.data)
image_for_upgrade = ':'.join([image_url, new_image_tag])
log.info(f"Image: {image_for_upgrade} will be used for upgrade.")
cs_data['spec']['image'] = image_for_upgrade
with NamedTemporaryFile() as cs_yaml:
dump_data_to_temp_yaml(cs_data, cs_yaml.name)
ocs_catalog.apply(cs_yaml.name)
# Wait for the new package manifest for upgrade.
operator_selector = get_selector_for_ocs_operator()
package_manifest = PackageManifest(
resource_name=OCS_OPERATOR_NAME,
selector=operator_selector,
)
package_manifest.wait_for_resource()
channel = config.DEPLOYMENT.get('ocs_csv_channel')
if not channel:
channel = package_manifest.get_default_channel()
# update subscription
subscription = OCP(
resource_name=constants.OCS_SUBSCRIPTION,
kind='subscription',
namespace=config.ENV_DATA['cluster_namespace'],
)
current_ocs_source = subscription.data['spec']['source']
log.info(f"Current OCS subscription source: {current_ocs_source}")
ocs_source = current_ocs_source if upgrade_in_current_source else (
constants.OPERATOR_CATALOG_SOURCE_NAME)
patch_subscription_cmd = (
f'oc patch subscription {constants.OCS_SUBSCRIPTION} '
f'-n {namespace} --type merge -p \'{{"spec":{{"channel": '
f'"{channel}", "source": "{ocs_source}"}}}}\'')
run_cmd(patch_subscription_cmd)
subscription_plan_approval = config.DEPLOYMENT.get(
'subscription_plan_approval')
if subscription_plan_approval == 'Manual':
wait_for_install_plan_and_approve(namespace)
attempts = 145
for attempt in range(1, attempts + 1):
log.info(f"Attempt {attempt}/{attempts} to check CSV upgraded.")
csv_name_post_upgrade = package_manifest.get_current_csv(channel)
if csv_name_post_upgrade == csv_name_pre_upgrade:
log.info(f"CSV is still: {csv_name_post_upgrade}")
sleep(5)
else:
log.info(f"CSV now upgraded to: {csv_name_post_upgrade}")
break
if attempts == attempt:
raise TimeoutException("No new CSV found after upgrade!")
csv_post_upgrade = CSV(resource_name=csv_name_post_upgrade,
namespace=namespace)
log.info(
f"Waiting for CSV {csv_name_post_upgrade} to be in succeeded state"
)
if version_before_upgrade == '4.2' and upgrade_version == '4.3':
log.info("Force creating Ceph toolbox after upgrade 4.2 -> 4.3")
setup_ceph_toolbox(force_setup=True)
osd_count = get_osd_count()
csv_post_upgrade.wait_for_phase("Succeeded", timeout=200 * osd_count)
post_upgrade_images = get_images(csv_post_upgrade.get())
old_images, _, _ = get_upgrade_image_info(pre_upgrade_images,
post_upgrade_images)
verify_image_versions(old_images, parsed_upgrade_version)
ocs_install_verification(
timeout=600,
skip_osd_distribution_check=True,
ocs_registry_image=ocs_registry_image,
post_upgrade_verification=True,
)
def test_upgrade():
ceph_cluster = CephCluster()
with CephHealthMonitor(ceph_cluster):
namespace = config.ENV_DATA['cluster_namespace']
ocs_catalog = CatalogSource(
resource_name=constants.OPERATOR_CATALOG_SOURCE_NAME,
namespace=constants.MARKETPLACE_NAMESPACE,
)
version_before_upgrade = config.ENV_DATA.get("ocs_version")
upgrade_version = config.UPGRADE.get("upgrade_ocs_version",
version_before_upgrade)
parsed_version_before_upgrade = parse_version(version_before_upgrade)
parsed_upgrade_version = parse_version(upgrade_version)
assert parsed_upgrade_version >= parsed_version_before_upgrade, (
f"Version you would like to upgrade to: {upgrade_version} "
f"is not higher or equal to the version you currently running: "
f"{version_before_upgrade}")
version_change = parsed_upgrade_version > parsed_version_before_upgrade
if version_change:
version_config_file = os.path.join(constants.CONF_DIR,
'ocs_version',
f'ocs-{upgrade_version}.yaml')
assert os.path.exists(version_config_file), (
f"OCS version config file {version_config_file} doesn't exist!"
)
with open(os.path.abspath(
os.path.expanduser(version_config_file))) as file_stream:
custom_config_data = yaml.safe_load(file_stream)
config.update(custom_config_data)
image_url = ocs_catalog.get_image_url()
image_tag = ocs_catalog.get_image_name()
log.info(f"Current image is: {image_url}, tag: {image_tag}")
ocs_registry_image = config.UPGRADE.get('upgrade_ocs_registry_image')
if ocs_registry_image:
image_url, new_image_tag = ocs_registry_image.split(':')
elif config.UPGRADE.get('upgrade_to_latest', True) or version_change:
new_image_tag = get_latest_ds_olm_tag()
else:
new_image_tag = get_next_version_available_for_upgrade(image_tag)
cs_data = deepcopy(ocs_catalog.data)
image_for_upgrade = ':'.join([image_url, new_image_tag])
log.info(f"Image: {image_for_upgrade} will be used for upgrade.")
cs_data['spec']['image'] = image_for_upgrade
operator_selector = get_selector_for_ocs_operator()
package_manifest = PackageManifest(
resource_name=OCS_OPERATOR_NAME,
selector=operator_selector,
)
csv_name_pre_upgrade = package_manifest.get_current_csv()
log.info(f"CSV name before upgrade is: {csv_name_pre_upgrade}")
csv_pre_upgrade = CSV(resource_name=csv_name_pre_upgrade,
namespace=namespace)
pre_upgrade_images = get_images(csv_pre_upgrade.get())
with NamedTemporaryFile() as cs_yaml:
dump_data_to_temp_yaml(cs_data, cs_yaml.name)
ocs_catalog.apply(cs_yaml.name)
# Wait for package manifest is ready
package_manifest.wait_for_resource()
subscription_plan_approval = config.DEPLOYMENT.get(
'subscription_plan_approval')
if subscription_plan_approval == 'Manual':
wait_for_install_plan_and_approve(namespace)
attempts = 145
for attempt in range(1, attempts):
if attempts == attempt:
raise TimeoutException("No new CSV found after upgrade!")
log.info(f"Attempt {attempt}/{attempts} to check CSV upgraded.")
package_manifest.reload_data()
csv_name_post_upgrade = package_manifest.get_current_csv()
if csv_name_post_upgrade == csv_name_pre_upgrade:
log.info(f"CSV is still: {csv_name_post_upgrade}")
sleep(5)
else:
log.info(f"CSV now upgraded to: {csv_name_post_upgrade}")
break
csv_post_upgrade = CSV(resource_name=csv_name_post_upgrade,
namespace=namespace)
log.info(
f"Waiting for CSV {csv_name_post_upgrade} to be in succeeded state"
)
if version_before_upgrade == '4.2' and upgrade_version == '4.3':
log.info("Force creating Ceph toolbox after upgrade 4.2 -> 4.3")
setup_ceph_toolbox(force_setup=True)
csv_post_upgrade.wait_for_phase("Succeeded", timeout=600)
post_upgrade_images = get_images(csv_post_upgrade.get())
old_images, _, _ = get_upgrade_image_info(pre_upgrade_images,
post_upgrade_images)
verify_image_versions(old_images, parsed_upgrade_version)
ocs_install_verification(timeout=600, skip_osd_distribution_check=True)