def get_job_dict(): """ Job template for fio workloads. Returns: dict: YAML data for a job object """ arch = get_system_architecture() if arch.startswith("x86"): image = "quay.io/fbalak/fio-fedora:latest" else: image = "quay.io/multi-arch/fio-fedora:latest" log.info(f"Discovered architecture: {arch.strip()}") log.info(f"Using image: {image}") # TODO(fbalak): load dictionary fixtures from one place template = textwrap.dedent(f""" apiVersion: batch/v1 kind: Job metadata: name: fio spec: backoffLimit: 0 template: metadata: name: fio spec: containers: - name: fio image: {image} command: - "/usr/bin/fio" - "--output-format=json" - "/etc/fio/workload.fio" volumeMounts: - name: fio-target mountPath: /mnt/target - name: fio-config-volume mountPath: /etc/fio restartPolicy: Never volumes: - name: fio-target persistentVolumeClaim: claimName: fio-target - name: fio-config-volume configMap: name: fio-config """) job_dict = yaml.safe_load(template) # overwrite used image (required for disconnected installation) update_container_with_mirrored_image(job_dict) return job_dict
def rsync(src, dst, node, dst_node=True, extra_params=""): """ This function will rsync source folder to destination path. You can rsync local folder to the node or vice versa depends on dst_node parameter. By default the rsync is from local to the node. Args: src (str): Source path of folder to rsync. dst (str): Destination path where to rsync. node (str): Node to/from copy. dst_node (bool): True if the destination (dst) is the node, False when dst is the local folder. extra_params (str): "See: oc rsync --help for the extra params" """ pod_name = f"rsync-{node.replace('.', '-')}" pod_data = load_yaml(constants.RSYNC_POD_YAML) pod_data["metadata"]["name"] = pod_name pod_data["spec"]["nodeName"] = node update_container_with_mirrored_image(pod_data) pod = OCP(kind="pod", namespace=constants.DEFAULT_NAMESPACE) src = src if dst_node else f"{pod_name}:/host{src}" dst = f"{pod_name}:/host{dst}" if dst_node else dst try: with tempfile.NamedTemporaryFile() as rsync_pod_yaml: dump_data_to_temp_yaml(pod_data, rsync_pod_yaml.name) pod.create(yaml_file=rsync_pod_yaml.name) pod.wait_for_resource(condition=constants.STATUS_RUNNING, timeout=120) rsync_cmd = f"rsync {extra_params} {src} {dst}" out = pod.exec_oc_cmd(rsync_cmd) log.info(f"Rsync out: {out}") finally: try: pod.delete(resource_name=pod_name) except CommandFailed: log.warning(f"Pod {pod_name} wasn't successfully deleted!") raise
def log_reader_writer_parallel(self): """ Write and read logfile stored on cephfs volume, from all worker nodes of a cluster via k8s Deployment. Raise: NotFoundError: When given volume is not found in given spec UnexpectedBehaviour: When an unexpected problem with starting the workload occurred """ # get deployment dict for the reproducer logwriter workload with open(constants.LOGWRITER_CEPHFS_REPRODUCER, "r") as deployment_file: self.deploy_dict = yaml.safe_load(deployment_file.read()) # if we are running in disconnected environment, we need to mirror the # container image first, and then use the mirror instead of the original if config.DEPLOYMENT.get("disconnected"): update_container_with_mirrored_image( self.deploy_dict["spec"]["template"]) # we need to match deployment replicas with number of worker nodes self.deploy_dict["spec"]["replicas"] = len(get_worker_nodes()) # drop topology spread constraints related to zones topology.drop_topology_constraint( self.deploy_dict["spec"]["template"]["spec"], topology.ZONE_LABEL) # and link the deployment with the pvc try: link_spec_volume( self.deploy_dict["spec"]["template"]["spec"], "logwriter-cephfs-volume", self.pvc_dict["metadata"]["name"], ) except (exceptions.NotFoundError, KeyError) as ex: logger.warning( "Failed to link the deployment with the pvc. We may need to check if the " "LOGWRITER_CEPHFS_REPRODUCER still matches the code of this test" ) raise ex # prepare k8s yaml file for deployment self.workload_file = ObjectConfFile( "log_reader_writer_parallel", [self.pvc_dict, self.deploy_dict], self.project, self.tmp_path, ) # deploy the workload, starting the log reader/writer pods logger.info( "starting log reader/writer workload via Deployment, one pod per worker" ) self.workload_file.create() logger.info("waiting for all pods of the workload Deployment to run") self.ocp_pod = ocp.OCP(kind="Pod", namespace=self.project.namespace) try: self.ocp_pod.wait_for_resource( resource_count=self.deploy_dict["spec"]["replicas"], condition=constants.STATUS_RUNNING, error_condition=constants.STATUS_ERROR, timeout=300, sleep=30, ) except Exception as ex: # this is not a problem with feature under test, but with infra, # cluster configuration or unrelated bug which must have happened # before this test case error_msg = "unexpected problem with start of the workload, cluster is either misconfigured or broken" logger.exception(error_msg) logger.debug(self.workload_file.describe()) raise exceptions.UnexpectedBehaviour(error_msg) from ex
def fetch_and_validate_data(self): """ While the workload is running, try to validate the data from the cephfs volume of the workload. Raise: NotFoundError: When the given volume is not found in given spec Exception: When the data verification job failed """ # if no obvious problem was detected, run the logreader job to validate # checksums in the log files (so that we are 100% sure that nothing went # wrong with the IO or the data) with open(constants.LOGWRITER_CEPHFS_READER, "r") as job_file: job_dict = yaml.safe_load(job_file.read()) # if we are running in disconnected environment, we need to mirror the # container image first, and then use the mirror instead of the original if config.DEPLOYMENT.get("disconnected"): update_container_with_mirrored_image( self.deploy_dict["spec"]["template"]) # drop topology spread constraints related to zones topology.drop_topology_constraint(job_dict["spec"]["template"]["spec"], topology.ZONE_LABEL) # we need to match number of jobs with the number used in the workload job_dict["spec"]["completions"] = self.deploy_dict["spec"]["replicas"] job_dict["spec"]["parallelism"] = self.deploy_dict["spec"]["replicas"] # and reffer to the correct pvc name try: link_spec_volume( job_dict["spec"]["template"]["spec"], "logwriter-cephfs-volume", self.pvc_dict["metadata"]["name"], ) except (exceptions.NotFoundError, KeyError) as ex: logger.warning( "Failed to link the deployment with the pvc. We may need to check if the " "LOGWRITER_CEPHFS_REPRODUCER still matches the code of this test" ) raise ex # prepare k8s yaml file for the job job_file = ObjectConfFile("log_reader", [job_dict], self.project, self.tmp_path) # deploy the job, starting the log reader pods logger.info( "starting log reader data validation job to fully check the log data", ) job_file.create() # wait for the logreader job to complete (this should be rather quick) try: job.wait_for_job_completion( job_name=job_dict["metadata"]["name"], namespace=self.project.namespace, timeout=300, sleep_time=30, ) except exceptions.TimeoutExpiredError: error_msg = "verification failed to complete in time: probably data loss or broken cluster" raise Exception(error_msg) # and then check that the job completed with success logger.info("checking the result of data validation job") logger.debug(job_file.describe()) ocp_job = ocp.OCP( kind="Job", namespace=self.project.namespace, resource_name=job_dict["metadata"]["name"], ) job_status = ocp_job.get()["status"] logger.info("last status of data verification job: %s", job_status) if ("failed" in job_status or job_status["succeeded"] != self.deploy_dict["spec"]["replicas"]): error_msg = "possible data corruption: data verification job failed!" logger.error(error_msg) job.log_output_of_job_pods(job_name=job_dict["metadata"]["name"], namespace=self.project.namespace) raise Exception(error_msg)
def test_log_reader_writer_parallel(project, tmp_path): """ Write and read logfile stored on cephfs volume, from all worker nodes of a cluster via k8s Deployment, while fetching content of the stored data via oc rsync to check the data locally. Reproduces BZ 1989301. Test failure means new blocker high priority bug. """ pvc_dict = get_pvc_dict() # we need to mount the volume on every worker node, so RWX/cephfs pvc_dict["metadata"]["name"] = "logwriter-cephfs-many" pvc_dict["spec"]["accessModes"] = [constants.ACCESS_MODE_RWX] if ( config.ENV_DATA["platform"].lower() not in constants.MANAGED_SERVICE_PLATFORMS ) and storagecluster_independent_check(): sc_name = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_CEPHFS else: sc_name = constants.CEPHFILESYSTEM_SC pvc_dict["spec"]["storageClassName"] = sc_name # there is no need for lot of storage capacity for this test pvc_dict["spec"]["resources"]["requests"]["storage"] = "1Gi" # get deployment dict for the reproducer logwriter workload with open(constants.LOGWRITER_CEPHFS_REPRODUCER, "r") as deployment_file: deploy_dict = yaml.safe_load(deployment_file.read()) # if we are running in disconnected environment, we need to mirror the # container image first, and then use the mirror instead of the original if config.DEPLOYMENT.get("disconnected"): update_container_with_mirrored_image(deploy_dict["spec"]["template"]) # we need to match deployment replicas with number of worker nodes deploy_dict["spec"]["replicas"] = len(get_worker_nodes()) # drop topology spread constraints related to zones topology.drop_topology_constraint( deploy_dict["spec"]["template"]["spec"], topology.ZONE_LABEL ) # and link the deployment with the pvc try: link_spec_volume( deploy_dict["spec"]["template"]["spec"], "logwriter-cephfs-volume", pvc_dict["metadata"]["name"], ) except Exception as ex: error_msg = "LOGWRITER_CEPHFS_REPRODUCER no longer matches code of this test" raise Exception(error_msg) from ex # prepare k8s yaml file for deployment workload_file = ObjectConfFile( "log_reader_writer_parallel", [pvc_dict, deploy_dict], project, tmp_path ) # deploy the workload, starting the log reader/writer pods logger.info( "starting log reader/writer workload via Deployment, one pod per worker" ) workload_file.create() logger.info("waiting for all pods of the workload Deployment to run") ocp_pod = ocp.OCP(kind="Pod", namespace=project.namespace) try: ocp_pod.wait_for_resource( resource_count=deploy_dict["spec"]["replicas"], condition=constants.STATUS_RUNNING, error_condition=constants.STATUS_ERROR, timeout=300, sleep=30, ) except Exception as ex: # this is not a problem with feature under test, but with infra, # cluster configuration or unrelated bug which must have happened # before this test case error_msg = "unexpected problem with start of the workload, cluster is either misconfigured or broken" logger.exception(error_msg) logger.debug(workload_file.describe()) raise exceptions.UnexpectedBehaviour(error_msg) from ex # while the workload is running, we will try to fetch and validate data # from the cephfs volume of the workload 120 times (this number of retries # is a bit larger than usual number required to reproduce bug from # BZ 1989301, but we need to be sure here) number_of_fetches = 120 # if given fetch fail, we will ignore the failure unless the number of # failures is too high (this has no direct impact on feature under test, # we should be able to detect the bug even with 10% of rsync failures, # since data corruption doesn't simply go away ...) number_of_failures = 0 allowed_failures = 12 is_local_data_ok = True local_dir = tmp_path / "logwriter" local_dir.mkdir() workload_pods = ocp_pod.get() workload_pod_name = workload_pods["items"][0]["metadata"]["name"] logger.info( "while the workload is running, we will fetch and check data from the cephfs volume %d times", number_of_fetches, ) for _ in range(number_of_fetches): # fetch data from cephfs volume into the local dir oc_cmd = [ "oc", "rsync", "--loglevel=4", "-n", project.namespace, f"pod/{workload_pod_name}:/mnt/target", local_dir, ] try: run_cmd(cmd=oc_cmd, timeout=300) except Exception as ex: number_of_failures += 1 # in case this fails, we are going to fetch extra evidence, that # said such failure is most likely related to OCP or infrastructure error_msg = "oc rsync failed: something is wrong with the cluster" logger.exception(error_msg) logger.debug(workload_file.describe()) oc_rpm_debug = [ "oc", "rsh", "-n", project.namespace, f"pod/{workload_pod_name}", "bash", "-c", ";".join( [ "rpm -qa", "rpm -qaV", "type -a tar", "tar --version", "type -a rsync", "rsync --version", ] ), ] try: run_cmd(cmd=oc_rpm_debug, timeout=600) except Exception: # if fetch of additional evidence fails, log and ignore the # exception (so that we can retry if needed) logger.exception("failed to fetch additional evidence") # in case the rsync run failed because of a container restart, # we assume the pod name hasn't changed, and just wait for the # container to be running again - unless the number of rsync # failures is too high if number_of_failures > allowed_failures: logger.error("number of ignored rsync failures is too high") else: ocp_pod.wait_for_resource( resource_count=deploy_dict["spec"]["replicas"], condition=constants.STATUS_RUNNING, error_condition=constants.STATUS_ERROR, timeout=300, sleep=30, ) continue logger.debug( "before this failure, we ignored %d previous failures", number_of_failures, ) raise exceptions.UnexpectedBehaviour(error_msg) from ex # look for null bytes in the just fetched local files in target dir, # and if these binary bytes are found, the test failed (the bug # was reproduced) target_dir = os.path.join(local_dir, "target") for file_name in os.listdir(target_dir): with open(os.path.join(target_dir, file_name), "r") as fo: data = fo.read() if "\0" in data: is_local_data_ok = False logger.error( "file %s is corrupted: null byte found in a text file", file_name, ) # is_local_data_ok = False assert is_local_data_ok, "data corruption detected" time.sleep(2) logger.debug("number of ignored rsync failures: %d", number_of_failures) # if no obvious problem was detected, run the logreader job to validate # checksums in the log files (so that we are 100% sure that nothing went # wrong with the IO or the data) with open(constants.LOGWRITER_CEPHFS_READER, "r") as job_file: job_dict = yaml.safe_load(job_file.read()) # mirroring for disconnected environment, if necessary if config.DEPLOYMENT.get("disconnected"): update_container_with_mirrored_image(job_dict["spec"]["template"]) # drop topology spread constraints related to zones topology.drop_topology_constraint( job_dict["spec"]["template"]["spec"], topology.ZONE_LABEL ) # we need to match number of jobs with the number used in the workload job_dict["spec"]["completions"] = deploy_dict["spec"]["replicas"] job_dict["spec"]["parallelism"] = deploy_dict["spec"]["replicas"] # and reffer to the correct pvc name try: link_spec_volume( job_dict["spec"]["template"]["spec"], "logwriter-cephfs-volume", pvc_dict["metadata"]["name"], ) except Exception as ex: error_msg = "LOGWRITER_CEPHFS_READER no longer matches code of this test" raise Exception(error_msg) from ex # prepare k8s yaml file for the job job_file = ObjectConfFile("log_reader", [job_dict], project, tmp_path) # deploy the job, starting the log reader pods logger.info( "starting log reader data validation job to fully check the log data", ) job_file.create() # wait for the logreader job to complete (this should be rather quick) try: job.wait_for_job_completion( job_name=job_dict["metadata"]["name"], namespace=project.namespace, timeout=300, sleep_time=30, ) except exceptions.TimeoutExpiredError: error_msg = ( "verification failed to complete in time: data loss or broken cluster?" ) logger.exception(error_msg) # and then check that the job completed with success logger.info("checking the result of data validation job") logger.debug(job_file.describe()) ocp_job = ocp.OCP( kind="Job", namespace=project.namespace, resource_name=job_dict["metadata"]["name"], ) job_status = ocp_job.get()["status"] logger.info("last status of data verification job: %s", job_status) if ( "failed" in job_status or job_status["succeeded"] != deploy_dict["spec"]["replicas"] ): error_msg = "possible data corruption: data verification job failed!" logger.error(error_msg) job.log_output_of_job_pods( job_name=job_dict["metadata"]["name"], namespace=project.namespace ) raise Exception(error_msg)