class TestSmallFileWorkload(PASTest): """ Deploy benchmark operator and run SmallFile workload SmallFile workload using https://github.com/distributed-system-analysis/smallfile smallfile is a python-based distributed POSIX workload generator which can be used to quickly measure performance for a variety of metadata-intensive workloads """ def setup(self): """ Setting up test parameters """ log.info("Starting the test setup") self.benchmark_name = "SmallFiles" self.client_pod_name = "smallfile-client" if config.PERF.get("deploy_internal_es"): self.es = ElasticSearch() else: if config.PERF.get("internal_es_server") == "": self.es = None return else: self.es = { "server": config.PERF.get("internal_es_server"), "port": config.PERF.get("internal_es_port"), "url": f"http://{config.PERF.get('internal_es_server')}:{config.PERF.get('internal_es_port')}", } # verify that the connection to the elasticsearch server is OK if not super(TestSmallFileWorkload, self).es_connect(): self.es = None return super(TestSmallFileWorkload, self).setup() # deploy the benchmark-operator self.deploy_benchmark_operator() def setting_storage_usage(self, file_size, files, threads, samples, clients): """ Getting the storage capacity, calculate the usage of the storage and setting the workload CR rile parameters. Args: file_size (int) : the size of the file to be used files (int) : number of files to use threads (int) : number of threads to be use in the test samples (int) : how meany samples to run for each test clients (int) : number of clients (pods) to use in the test """ self.crd_data["spec"]["workload"]["args"]["file_size"] = file_size self.crd_data["spec"]["workload"]["args"]["files"] = files self.crd_data["spec"]["workload"]["args"]["threads"] = threads self.crd_data["spec"]["workload"]["args"]["samples"] = samples self.crd_data["spec"]["workload"]["args"]["clients"] = clients # Calculating the size of the volume that need to be test, it should # be at least twice in the size then the size of the files, and at # least 100Gi. # Since the file_size is in Kb and the vol_size need to be in Gb, more # calculation is needed. vol_size = int(files * threads * file_size * 3) vol_size = int(vol_size / constants.GB2KB) if vol_size < 100: vol_size = 100 self.crd_data["spec"]["workload"]["args"][ "storagesize"] = f"{vol_size}Gi" def init_full_results(self, full_results): """ Initialize the full results object which will send to the ES server Args: full_results (obj): an empty SmallFileResultsAnalyse object Returns: SmallFileResultsAnalyse (obj): the input object fill with data """ for key in self.environment: full_results.add_key(key, self.environment[key]) # Calculating the total size of the working data set - in GB full_results.add_key( "dataset", self.crd_data["spec"]["workload"]["args"]["file_size"] * self.crd_data["spec"]["workload"]["args"]["files"] * self.crd_data["spec"]["workload"]["args"]["threads"] * full_results.results["clients"] / constants.GB2KB, ) full_results.add_key( "global_options", { "files": self.crd_data["spec"]["workload"]["args"]["files"], "file_size": self.crd_data["spec"]["workload"]["args"]["file_size"], "storageclass": self.crd_data["spec"]["workload"]["args"]["storageclass"], "vol_size": self.crd_data["spec"]["workload"]["args"]["storagesize"], }, ) return full_results def generate_kibana_link(self, index, columns): """ Generating full link to the Kibana server with full test results information Args: index (str): the kibana index name (results, response time, etc.) columns (str): list of all columns to display Return: str : an http link to the appropriate kibana report """ stime = self.start_time.replace("GMT", ".000Z") etime = self.end_time.replace("GMT", ".000Z") log.info( json.dumps(self.crd_data.get("spec").get("elasticsearch"), indent=2)) host = self.crd_data.get("spec").get("elasticsearch").get("url") try: host = host.split(":")[1].replace("//", "") except Exception: log.error("No ES configuretion") return "" kibana_id = self.get_kibana_indexid(host, index) app = "app/kibana#/discover" if self.dev_mode: app = "app/discover#/" result = ( f"http://{host}:5601/{app}" f"?_a=(columns:!({columns}),filters:!(),index:'{kibana_id}',interval:auto," f"query:(language:kuery,query:'uuid:{self.uuid}'),sort:!())" f"&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:'{stime}',to:'{etime}'))" ) return result def collect_benchmark_logs(self): """ Collecting the test log from all benchmark pods """ # Getting full list of benchmark clients self.full_client_list = get_pod_name_by_pattern( self.client_pod_name, benchmark_operator.BMO_NAME) # Collecting logs from each pod for clpod in self.full_client_list: test_logs = self.pod_obj.exec_oc_cmd(f"logs {clpod}", out_yaml_format=False) log_file_name = f"{self.full_log_path}/{clpod}-pod.log" try: with open(log_file_name, "w") as f: f.write(test_logs) log.info(f"The Test log can be found at : {log_file_name}") except Exception: log.warning( f"Cannot write the log to the file {log_file_name}") log.info("Logs from all client pods got successfully") def run(self): log.info("Running SmallFile bench") self.deploy_and_wait_for_wl_to_start(timeout=240, sleep=10) # Getting the UUID from inside the benchmark pod self.uuid = self.operator.get_uuid(self.client_pod) self.wait_for_wl_to_finish(sleep=30) self.collect_benchmark_logs() try: if "RUN STATUS DONE" in self.test_logs: log.info("SmallFiles has completed successfully") return True except IOError: log.warning("SmallFiles failed to complete") return False def teardown(self): """ The teardown of the test environment in the end. """ log.info("cleanup the environment") if isinstance(self.es, ElasticSearch): self.es.cleanup() self.operator.cleanup() # wait up to 45 min for the ceph cluster be health OK after backend # operation completed. log.info("Verify (and wait if needed) that ceph health is OK") ceph_health_check(tries=45, delay=60) # Let the background operation (delete backed images) to finish time.sleep(120) @pytest.mark.parametrize( argnames=[ "file_size", "files", "threads", "samples", "clients", "interface" ], argvalues=[ pytest.param(*[4, 5000, 22, 5, 33, constants.CEPHBLOCKPOOL]), pytest.param(*[16, 5000, 8, 5, 21, constants.CEPHBLOCKPOOL]), pytest.param(*[4, 2500, 4, 5, 9, constants.CEPHFILESYSTEM]), pytest.param(*[16, 1500, 4, 5, 9, constants.CEPHFILESYSTEM]), ], ) @pytest.mark.polarion_id("OCS-1295") def test_smallfile_workload(self, file_size, files, threads, samples, clients, interface): """ Run SmallFile Workload Args: file_size (int) : the size of the file to be used files (int) : number of files to use threads (int) : number of threads to be use in the test samples (int) : how meany samples to run for each test interface (str) : the volume type (rbd / cephfs) """ # verify that there is an elasticsearch server for the benchmark if not self.es: log.error("This test must have an Elasticsearch server") return False # Getting the full path for the test logs self.full_log_path = get_full_test_logs_path(cname=self) self.results_path = get_full_test_logs_path(cname=self) self.full_log_path += ( f"-{file_size}-{files}-{threads}-{samples}-{clients}-{interface}") log.info(f"Logs file path name is : {self.full_log_path}") # Loading the main template yaml file for the benchmark log.info("Create resource file for small_files workload") self.crd_data = templating.load_yaml( constants.SMALLFILE_BENCHMARK_YAML) # Saving the Original elastic-search IP and PORT - if defined in yaml self.es_info_backup(self.es) self.set_storageclass(interface=interface) # Setting the data set to 40% of the total storage capacity self.setting_storage_usage(file_size, files, threads, samples, clients) self.get_env_info() if not self.run(): log.error("The benchmark failed to run !") return # Setting back the original elastic-search information if self.backup_es: self.crd_data["spec"]["elasticsearch"] = self.backup_es # Initialize the results doc file. full_results = self.init_full_results( SmallFileResultsAnalyse(self.uuid, self.crd_data, self.full_log_path, self.main_es)) log.info(f"Full results is : {full_results.results}") if isinstance(self.es, ElasticSearch): # Using internal deployed elasticsearch log.info("Getting data from internal ES") if self.main_es: self.copy_es_data(self.es) full_results.read() else: log.info("Dumping data from the Internal ES to tar ball file") self.es.dumping_all_data(self.full_log_path) else: log.info(self.es) self.es = Elasticsearch(hosts=[{ "host": self.es["server"], "port": self.es["port"] }]) full_results.read() full_results.add_key("test_time", { "start": self.start_time, "end": self.end_time }) if self.main_es: full_results.es = self.main_es if not full_results.dont_check: full_results.add_key("hosts", full_results.get_clients_list()) full_results.init_full_results() full_results.aggregate_host_results() test_status = full_results.aggregate_samples_results() # Generate link for the all data in the kibana columens = "optype,files,filesPerSec,elapsed,sample,tid" klink = self.generate_kibana_link("ripsaw-smallfile-results", columens) # Generate link for the all response-time data in the kibana columens = "optype,sample,iops,max,min,mean,'90%25','95%25','99%25'" rtlink = self.generate_kibana_link("ripsaw-smallfile-rsptimes", columens) full_results.all_results = { "kibana_all": klink, "kibana_rsptime": rtlink } if full_results.es_write(): res_link = full_results.results_link() log.info(f"The Result can be found at : {res_link}") # Create text file with results of all subtest (4 - according to the parameters) self.write_result_to_file(res_link) else: test_status = True assert test_status, "Test Failed !" def test_smallfile_results(self): """ This is not a test - it is only check that previous test ran and finish as expected and reporting the full results (links in the ES) of previous tests (4) """ # TODO : This function will push the results (if exists) to the performance dashboard. self.results_path = get_full_test_logs_path( cname=self, fname="test_smallfile_workload") self.results_file = os.path.join(self.results_path, "all_results.txt") log.info(f"Check results in {self.results_file}") try: input_file = open(self.results_file, "r") data = input_file.read().split("\n") data.pop() # remove the last empty element input_file.close() if len(data) != 4: log.error("Not all tests finished") raise exceptions.BenchmarkTestFailed() else: log.info( "All test finished OK, and the results can be found at :") for res in data: log.info(res) except OSError as err: log.error(f"OS error: {err}") raise err
class TestSmallFileWorkload(PASTest): """ Deploy benchmark operator and run SmallFile workload SmallFile workload using https://github.com/distributed-system-analysis/smallfile smallfile is a python-based distributed POSIX workload generator which can be used to quickly measure performance for a variety of metadata-intensive workloads """ def setup(self): """ Setting up test parameters """ log.info("Starting the test setup") self.benchmark_name = "SmallFiles" self.client_pod_name = "smallfile-client" if config.PERF.get("deploy_internal_es"): self.es = ElasticSearch() else: if config.PERF.get("internal_es_server") == "": self.es = None return else: self.es = { "server": config.PERF.get("internal_es_server"), "port": config.PERF.get("internal_es_port"), "url": f"http://{config.PERF.get('internal_es_server')}:{config.PERF.get('internal_es_port')}", } # verify that the connection to the elasticsearch server is OK if not super(TestSmallFileWorkload, self).es_connect(): self.es = None return super(TestSmallFileWorkload, self).setup() # deploy the benchmark-operator self.deploy_benchmark_operator() def setting_storage_usage(self, file_size, files, threads, samples): """ Getting the storage capacity, calculate the usage of the storage and setting the workload CR rile parameters. Args: file_size (int) : the size of the file to be used files (int) : number of files to use threads (int) : number of threads to be use in the test samples (int) : how meany samples to run for each test """ self.crd_data["spec"]["workload"]["args"]["file_size"] = file_size self.crd_data["spec"]["workload"]["args"]["files"] = files self.crd_data["spec"]["workload"]["args"]["threads"] = threads self.crd_data["spec"]["workload"]["args"]["samples"] = samples # Calculating the size of the volume that need to be test, it should # be at least twice in the size then the size of the files, and at # least 100Gi. # Since the file_size is in Kb and the vol_size need to be in Gb, more # calculation is needed. vol_size = int(files * threads * file_size * 3) vol_size = int(vol_size / constants.GB2KB) if vol_size < 100: vol_size = 100 self.crd_data["spec"]["workload"]["args"][ "storagesize"] = f"{vol_size}Gi" def init_full_results(self, full_results): """ Initialize the full results object which will send to the ES server Args: full_results (obj): an empty SmallFileResultsAnalyse object Returns: SmallFileResultsAnalyse (obj): the input object fill with data """ for key in self.environment: full_results.add_key(key, self.environment[key]) # Calculating the total size of the working data set - in GB full_results.add_key( "dataset", self.crd_data["spec"]["workload"]["args"]["file_size"] * self.crd_data["spec"]["workload"]["args"]["files"] * self.crd_data["spec"]["workload"]["args"]["threads"] * full_results.results["clients"] / constants.GB2KB, ) full_results.add_key( "global_options", { "files": self.crd_data["spec"]["workload"]["args"]["files"], "file_size": self.crd_data["spec"]["workload"]["args"]["file_size"], "storageclass": self.crd_data["spec"]["workload"]["args"]["storageclass"], "vol_size": self.crd_data["spec"]["workload"]["args"]["storagesize"], }, ) return full_results def run(self): log.info("Running SmallFile bench") self.deploy_and_wait_for_wl_to_start(timeout=240, sleep=10) # Getting the UUID from inside the benchmark pod self.uuid = self.operator.get_uuid(self.client_pod) self.wait_for_wl_to_finish(sleep=30) try: if "RUN STATUS DONE" in self.test_logs: log.info("SmallFiles has completed successfully") return True except IOError: log.warning("SmallFiles failed to complete") return False def teardown(self): """ The teardown of the test environment in the end. """ log.info("cleanup the environment") if isinstance(self.es, ElasticSearch): self.es.cleanup() self.operator.cleanup() # wait up to 45 min for the ceph cluster be health OK after backend # operation completed. log.info("Verify (and wait if needed) that ceph health is OK") ceph_health_check(tries=45, delay=60) @pytest.mark.parametrize( argnames=["file_size", "files", "threads", "samples", "interface"], argvalues=[ pytest.param( *[4, 50000, 4, 3, constants.CEPHBLOCKPOOL], marks=pytest.mark.polarion_id("OCS-1295"), ), pytest.param( *[16, 50000, 4, 3, constants.CEPHBLOCKPOOL], marks=pytest.mark.polarion_id("OCS-2020"), ), pytest.param( *[16, 200000, 4, 3, constants.CEPHBLOCKPOOL], marks=pytest.mark.polarion_id("OCS-2021"), ), pytest.param( *[4, 50000, 4, 3, constants.CEPHFILESYSTEM], marks=pytest.mark.polarion_id("OCS-2022"), ), pytest.param( *[16, 50000, 4, 3, constants.CEPHFILESYSTEM], marks=pytest.mark.polarion_id("OCS-2023"), ), ], ) @pytest.mark.polarion_id("OCS-1295") def test_smallfile_workload(self, file_size, files, threads, samples, interface): """ Run SmallFile Workload Args: file_size (int) : the size of the file to be used files (int) : number of files to use threads (int) : number of threads to be use in the test samples (int) : how meany samples to run for each test interface (str) : the volume type (rbd / cephfs) """ # verify that there is an elasticsearch server for the benchmark if not self.es: log.error("This test must have an Elasticsearch server") return False # Getting the full path for the test logs self.full_log_path = get_full_test_logs_path(cname=self) self.full_log_path += f"-{file_size}-{files}-{threads}-{samples}-{interface}" log.info(f"Logs file path name is : {self.full_log_path}") # Loading the main template yaml file for the benchmark log.info("Create resource file for smallfiles workload") self.crd_data = templating.load_yaml( constants.SMALLFILE_BENCHMARK_YAML) # Saving the Original elastic-search IP and PORT - if defined in yaml self.es_info_backup(self.es) self.set_storageclass(interface=interface) # Setting the data set to 40% of the total storage capacity self.setting_storage_usage(file_size, files, threads, samples) self.get_env_info() if not self.run(): log.error("The benchmark failed to run !") return # Setting back the original elastic-search information if self.backup_es: self.crd_data["spec"]["elasticsearch"] = self.backup_es # Initialize the results doc file. full_results = self.init_full_results( SmallFileResultsAnalyse(self.uuid, self.crd_data, self.full_log_path, self.main_es)) log.info(f"Full results is : {full_results.results}") if isinstance(self.es, ElasticSearch): # Using internal deployed elasticsearch log.info("Getting data from internal ES") if self.main_es: self.copy_es_data(self.es) full_results.read() else: log.info("Dumping data from the Internal ES to tar ball file") self.es.dumping_all_data(self.full_log_path) else: log.info(self.es) self.es = Elasticsearch(hosts=[{ "host": self.es["server"], "port": self.es["port"] }]) full_results.read() full_results.add_key("test_time", { "start": self.start_time, "end": self.end_time }) if self.main_es: full_results.es = self.main_es if not full_results.dont_check: full_results.add_key("hosts", full_results.get_clients_list()) full_results.init_full_results() full_results.aggregate_host_results() test_status = full_results.aggregate_samples_results() full_results.all_results = None if full_results.es_write(): log.info( f"The Result can be found at : {full_results.results_link()}" ) else: test_status = True assert test_status, "Test Failed !"