def verify_image_versions(old_images): """ Verify if all the images of OCS objects got upgraded Args: old_images (set): set with old images """ namespace = config.ENV_DATA['cluster_namespace'] number_of_worker_nodes = len(get_typed_nodes()) storage_cluster = StorageCluster( resource_name=config.ENV_DATA['storage_cluster_name'], namespace=namespace) osd_count = ( int(storage_cluster.data['spec']['storageDeviceSets'][0]['count']) * int(storage_cluster.data['spec']['storageDeviceSets'][0]['replica'])) verify_pods_upgraded(old_images, selector=constants.OCS_OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.NOOBAA_APP_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_CEPHFSPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_RBDPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_RBDPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded(old_images, selector=constants.MGR_APP_LABEL) verify_pods_upgraded(old_images, selector=constants.MON_APP_LABEL, count=3) verify_pods_upgraded(old_images, selector=constants.OSD_APP_LABEL, count=osd_count) verify_pods_upgraded(old_images, selector=constants.MDS_APP_LABEL, count=2) if config.ENV_DATA.get('platform') == constants.VSPHERE_PLATFORM: verify_pods_upgraded(old_images, selector=constants.RGW_APP_LABEL, count=1)
def verify_image_versions(old_images, upgrade_version): """ Verify if all the images of OCS objects got upgraded Args: old_images (set): set with old images upgrade_version (packaging.version.Version): version of OCS """ number_of_worker_nodes = len(get_typed_nodes()) osd_count = get_osd_count() verify_pods_upgraded(old_images, selector=constants.OCS_OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.OPERATOR_LABEL) # in 4.3 app selector nooba have those pods: noobaa-core-ID, noobaa-db-ID, # noobaa-operator-ID but in 4.2 only 2: noobaa-core-ID, noobaa-operator-ID nooba_pods = 2 if upgrade_version < parse_version('4.3') else 3 verify_pods_upgraded(old_images, selector=constants.NOOBAA_APP_LABEL, count=nooba_pods) verify_pods_upgraded( old_images, selector=constants.CSI_CEPHFSPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_RBDPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_RBDPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded(old_images, selector=constants.MON_APP_LABEL, count=3) verify_pods_upgraded(old_images, selector=constants.MGR_APP_LABEL) # OSD upgrade have timeout 10mins for new attempt if cluster is not health. # https://bugzilla.redhat.com/show_bug.cgi?id=1840729 setting timeout for # 12.5 minutes per OSD verify_pods_upgraded( old_images, selector=constants.OSD_APP_LABEL, count=osd_count, timeout=750 * osd_count, ) verify_pods_upgraded(old_images, selector=constants.MDS_APP_LABEL, count=2) if config.ENV_DATA.get('platform') in constants.ON_PREM_PLATFORMS: # Workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1857802 - RGW count is 1 # post upgrade to OCS 4.5. Tracked with # https://github.com/red-hat-storage/ocs-ci/issues/2532 # TODO: uncomment the below 1 line: # rgw_count = 2 if float(config.ENV_DATA['ocs_version']) >= 4.5 else 1 # TODO: Delete the below 1 line rgw_count = 1 verify_pods_upgraded(old_images, selector=constants.RGW_APP_LABEL, count=rgw_count)
def verify_image_versions(old_images, upgrade_version, version_before_upgrade): """ Verify if all the images of OCS objects got upgraded Args: old_images (set): set with old images upgrade_version (packaging.version.Version): version of OCS version_before_upgrade (float): version of OCS before upgrade """ number_of_worker_nodes = len(get_nodes()) verify_pods_upgraded(old_images, selector=constants.OCS_OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.OPERATOR_LABEL) default_noobaa_pods = 3 noobaa_pods = default_noobaa_pods if upgrade_version >= parse_version("4.7"): noobaa = OCP(kind="noobaa", namespace=config.ENV_DATA["cluster_namespace"]) resource = noobaa.get()["items"][0] endpoints = resource.get("spec", {}).get("endpoints", {}) max_endpoints = endpoints.get("maxCount", constants.MAX_NB_ENDPOINT_COUNT) min_endpoints = endpoints.get( "minCount", constants.MIN_NB_ENDPOINT_COUNT_POST_DEPLOYMENT) noobaa_pods = default_noobaa_pods + min_endpoints try: verify_pods_upgraded( old_images, selector=constants.NOOBAA_APP_LABEL, count=noobaa_pods, ) except TimeoutException as ex: if upgrade_version >= parse_version("4.7"): log.info( "Nooba pods didn't match. Trying once more with max noobaa endpoints!" f"Exception: {ex}") noobaa_pods = default_noobaa_pods + max_endpoints verify_pods_upgraded( old_images, selector=constants.NOOBAA_APP_LABEL, count=noobaa_pods, timeout=60, ) else: raise verify_pods_upgraded( old_images, selector=constants.CSI_CEPHFSPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_RBDPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_RBDPLUGIN_PROVISIONER_LABEL, count=2) if not config.DEPLOYMENT.get("external_mode"): verify_pods_upgraded( old_images, selector=constants.MON_APP_LABEL, count=3, ) verify_pods_upgraded(old_images, selector=constants.MGR_APP_LABEL) osd_timeout = 600 if upgrade_version >= parse_version("4.5") else 750 osd_count = get_osd_count() verify_pods_upgraded( old_images, selector=constants.OSD_APP_LABEL, count=osd_count, timeout=osd_timeout * osd_count, ) verify_pods_upgraded(old_images, selector=constants.MDS_APP_LABEL, count=2) if config.ENV_DATA.get("platform") in constants.ON_PREM_PLATFORMS: rgw_count = get_rgw_count(upgrade_version.base_version, True, version_before_upgrade) verify_pods_upgraded( old_images, selector=constants.RGW_APP_LABEL, count=rgw_count, )
def verify_image_versions(old_images, upgrade_version, version_before_upgrade): """ Verify if all the images of OCS objects got upgraded Args: old_images (set): set with old images upgrade_version (packaging.version.Version): version of OCS version_before_upgrade (float): version of OCS before upgrade """ number_of_worker_nodes = len(get_nodes()) osd_count = get_osd_count() verify_pods_upgraded(old_images, selector=constants.OCS_OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.OPERATOR_LABEL) # in 4.3 app selector nooba have those pods: noobaa-core-ID, noobaa-db-ID, # noobaa-operator-ID but in 4.2 only 2: noobaa-core-ID, noobaa-operator-ID nooba_pods = 2 if upgrade_version < parse_version("4.3") else 3 verify_pods_upgraded(old_images, selector=constants.NOOBAA_APP_LABEL, count=nooba_pods) verify_pods_upgraded( old_images, selector=constants.CSI_CEPHFSPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_RBDPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_RBDPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.MON_APP_LABEL, count=3, ) verify_pods_upgraded(old_images, selector=constants.MGR_APP_LABEL) osd_timeout = 600 if upgrade_version >= parse_version("4.5") else 750 verify_pods_upgraded( old_images, selector=constants.OSD_APP_LABEL, count=osd_count, timeout=osd_timeout * osd_count, ) verify_pods_upgraded(old_images, selector=constants.MDS_APP_LABEL, count=2) if config.ENV_DATA.get("platform") in constants.ON_PREM_PLATFORMS: rgw_count = get_rgw_count(upgrade_version.base_version, True, version_before_upgrade) verify_pods_upgraded( old_images, selector=constants.RGW_APP_LABEL, count=rgw_count, )
def verify_image_versions(old_images, upgrade_version, version_before_upgrade): """ Verify if all the images of OCS objects got upgraded Args: old_images (set): set with old images upgrade_version (packaging.version.Version): version of OCS version_before_upgrade (float): version of OCS before upgrade """ number_of_worker_nodes = len(get_nodes()) verify_pods_upgraded(old_images, selector=constants.OCS_OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.OPERATOR_LABEL) default_noobaa_pods = 3 noobaa_pods = default_noobaa_pods if upgrade_version >= parse_version("4.7"): noobaa = OCP(kind="noobaa", namespace=config.ENV_DATA["cluster_namespace"]) resource = noobaa.get()["items"][0] endpoints = resource.get("spec", {}).get("endpoints", {}) max_endpoints = endpoints.get("maxCount", constants.MAX_NB_ENDPOINT_COUNT) min_endpoints = endpoints.get( "minCount", constants.MIN_NB_ENDPOINT_COUNT_POST_DEPLOYMENT) noobaa_pods = default_noobaa_pods + min_endpoints try: verify_pods_upgraded( old_images, selector=constants.NOOBAA_APP_LABEL, count=noobaa_pods, ) except TimeoutException as ex: if upgrade_version >= parse_version("4.7"): log.info( "Nooba pods didn't match. Trying once more with max noobaa endpoints!" f"Exception: {ex}") noobaa_pods = default_noobaa_pods + max_endpoints verify_pods_upgraded( old_images, selector=constants.NOOBAA_APP_LABEL, count=noobaa_pods, timeout=60, ) else: raise verify_pods_upgraded( old_images, selector=constants.CSI_CEPHFSPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_RBDPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_RBDPLUGIN_PROVISIONER_LABEL, count=2) if not config.DEPLOYMENT.get("external_mode"): verify_pods_upgraded( old_images, selector=constants.MON_APP_LABEL, count=3, ) verify_pods_upgraded(old_images, selector=constants.MGR_APP_LABEL) osd_timeout = 600 if upgrade_version >= parse_version("4.5") else 750 osd_count = get_osd_count() # In the debugging issue: # https://github.com/red-hat-storage/ocs-ci/issues/5031 # Noticed that it's taking about 1 more minute from previous check till actual # OSD pods getting restarted. # Hence adding sleep here for 120 seconds to be sure, OSD pods upgrade started. log.info("Waiting for 2 minutes before start checking OSD pods") time.sleep(120) verify_pods_upgraded( old_images, selector=constants.OSD_APP_LABEL, count=osd_count, timeout=osd_timeout * osd_count, ) verify_pods_upgraded(old_images, selector=constants.MDS_APP_LABEL, count=2) if config.ENV_DATA.get("platform") in constants.ON_PREM_PLATFORMS: rgw_count = get_rgw_count(upgrade_version.base_version, True, version_before_upgrade) verify_pods_upgraded( old_images, selector=constants.RGW_APP_LABEL, count=rgw_count, ) if upgrade_version >= parse_version("4.6"): verify_pods_upgraded(old_images, selector=constants.OCS_METRICS_EXPORTER)
def verify_image_versions(old_images, upgrade_version): """ Verify if all the images of OCS objects got upgraded Args: old_images (set): set with old images upgrade_version (packaging.version.Version): version of OCS """ number_of_worker_nodes = len(get_typed_nodes()) osd_count = get_osd_count() verify_pods_upgraded(old_images, selector=constants.OCS_OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.OPERATOR_LABEL) # in 4.3 app selector nooba have those pods: noobaa-core-ID, noobaa-db-ID, # noobaa-operator-ID but in 4.2 only 2: noobaa-core-ID, noobaa-operator-ID nooba_pods = 2 if upgrade_version < parse_version('4.3') else 3 verify_pods_upgraded(old_images, selector=constants.NOOBAA_APP_LABEL, count=nooba_pods) verify_pods_upgraded( old_images, selector=constants.CSI_CEPHFSPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_RBDPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_RBDPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded(old_images, selector=constants.MON_APP_LABEL, count=3) verify_pods_upgraded(old_images, selector=constants.MGR_APP_LABEL) # OSD upgrade have timeout 10mins for new attempt if cluster is not health. # https://bugzilla.redhat.com/show_bug.cgi?id=1840729 setting timeout for # 12.5 minutes per OSD verify_pods_upgraded( old_images, selector=constants.OSD_APP_LABEL, count=osd_count, timeout=750 * osd_count, ) verify_pods_upgraded(old_images, selector=constants.MDS_APP_LABEL, count=2) if config.ENV_DATA.get('platform') == constants.VSPHERE_PLATFORM: verify_pods_upgraded(old_images, selector=constants.RGW_APP_LABEL, count=1)
def verify_image_versions(old_images, upgrade_version): """ Verify if all the images of OCS objects got upgraded Args: old_images (set): set with old images upgrade_version (packaging.version.Version): version of OCS """ namespace = config.ENV_DATA['cluster_namespace'] number_of_worker_nodes = len(get_typed_nodes()) storage_cluster = StorageCluster( resource_name=config.ENV_DATA['storage_cluster_name'], namespace=namespace) osd_count = ( int(storage_cluster.data['spec']['storageDeviceSets'][0]['count']) * int(storage_cluster.data['spec']['storageDeviceSets'][0]['replica'])) verify_pods_upgraded(old_images, selector=constants.OCS_OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.OPERATOR_LABEL) # in 4.3 app selector nooba have those pods: noobaa-core-ID, noobaa-db-ID, # noobaa-operator-ID but in 4.2 only 2: noobaa-core-ID, noobaa-operator-ID nooba_pods = 2 if upgrade_version < parse_version('4.3') else 3 verify_pods_upgraded(old_images, selector=constants.NOOBAA_APP_LABEL, count=nooba_pods) verify_pods_upgraded( old_images, selector=constants.CSI_CEPHFSPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_RBDPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_RBDPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded(old_images, selector=constants.MGR_APP_LABEL) verify_pods_upgraded(old_images, selector=constants.MON_APP_LABEL, count=3) verify_pods_upgraded(old_images, selector=constants.OSD_APP_LABEL, count=osd_count) verify_pods_upgraded(old_images, selector=constants.MDS_APP_LABEL, count=2) if config.ENV_DATA.get('platform') == constants.VSPHERE_PLATFORM: verify_pods_upgraded(old_images, selector=constants.RGW_APP_LABEL, count=1)
def verify_image_versions(old_images, upgrade_version, version_before_upgrade): """ Verify if all the images of OCS objects got upgraded Args: old_images (set): set with old images upgrade_version (packaging.version.Version): version of OCS version_before_upgrade (float): version of OCS before upgrade """ number_of_worker_nodes = len(get_nodes()) osd_count = get_osd_count() verify_pods_upgraded(old_images, selector=constants.OCS_OPERATOR_LABEL) verify_pods_upgraded(old_images, selector=constants.OPERATOR_LABEL) # in 4.3 app selector nooba have those pods: noobaa-core-ID, noobaa-db-ID, # noobaa-operator-ID but in 4.2 only 2: noobaa-core-ID, noobaa-operator-ID nooba_pods = 2 if upgrade_version < parse_version("4.3") else 3 verify_pods_upgraded(old_images, selector=constants.NOOBAA_APP_LABEL, count=nooba_pods) verify_pods_upgraded( old_images, selector=constants.CSI_CEPHFSPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.CSI_RBDPLUGIN_LABEL, count=number_of_worker_nodes, ) verify_pods_upgraded(old_images, selector=constants.CSI_RBDPLUGIN_PROVISIONER_LABEL, count=2) verify_pods_upgraded( old_images, selector=constants.MON_APP_LABEL, count=3, ) verify_pods_upgraded(old_images, selector=constants.MGR_APP_LABEL) osd_timeout = 600 if upgrade_version >= parse_version("4.5") else 750 verify_pods_upgraded( old_images, selector=constants.OSD_APP_LABEL, count=osd_count, timeout=osd_timeout * osd_count, ) verify_pods_upgraded(old_images, selector=constants.MDS_APP_LABEL, count=2) if config.ENV_DATA.get("platform") in constants.ON_PREM_PLATFORMS: # RGW count is 1 if the cluster was upgraded from <= 4.4 # Related bug - https://bugzilla.redhat.com/show_bug.cgi?id=1857802 rgw_count = 2 if float(version_before_upgrade) >= 4.5 else 1 verify_pods_upgraded( old_images, selector=constants.RGW_APP_LABEL, count=rgw_count, ) # With 4.4 OCS cluster deployed over Azure, RGW is the default backingstore if config.ENV_DATA.get("platform") == constants.AZURE_PLATFORM: if float(config.ENV_DATA["ocs_version"]) == 4.4 or ( float(config.ENV_DATA["ocs_version"]) == 4.5 and float(version_before_upgrade) < 4.5): rgw_count = 1 verify_pods_upgraded( old_images, selector=constants.RGW_APP_LABEL, count=rgw_count, )