def wait_for_nodes_status(node_names=None, status=constants.NODE_READY, timeout=180): """ Wait until all nodes are in the given status Args: node_names (list): The node names to wait for to reached the desired state If None, will wait for all cluster nodes status (str): The node status to wait for (e.g. 'Ready', 'NotReady', 'SchedulingDisabled') timeout (int): The number in seconds to wait for the nodes to reach the status Raises: ResourceWrongStatusException: In case one or more nodes haven't reached the desired state """ try: if not node_names: for sample in TimeoutSampler(60, 3, get_node_objs): if sample: node_names = [node.name for node in sample] break nodes_not_in_state = copy.deepcopy(node_names) log.info(f"Waiting for nodes {node_names} to reach status {status}") for sample in TimeoutSampler(timeout, 3, get_node_objs, nodes_not_in_state): for node in sample: if node.ocp.get_resource_status(node.name) == status: nodes_not_in_state.remove(node.name) if not nodes_not_in_state: break except TimeoutExpiredError: log.error( f"The following nodes haven't reached status {status}: {node_names}" ) raise exceptions.ResourceWrongStatusException( node_names, [n.describe() for n in get_node_objs(node_names)])
def __init__(self): """ Initializer function """ log.info("Initializing the Elastic-Search environment object") self.namespace = "elastic-system" self.eck_file = "ocs_ci/templates/app-pods/eck.1.6.0-all-in-one.yaml" self.dumper_file = "ocs_ci/templates/app-pods/esclient.yaml" self.crd = "ocs_ci/templates/app-pods/esq.yaml" # Creating some different types of OCP objects self.ocp = OCP(kind="pod", resource_name="elastic-operator-0", namespace=self.namespace) self.ns_obj = OCP(kind="namespace", namespace=self.namespace) self.es = OCP(resource_name="quickstart-es-http", namespace=self.namespace) self.elasticsearch = OCP(namespace=self.namespace, kind="elasticsearch") self.password = OCP( kind="secret", resource_name="quickstart-es-elastic-user", namespace=self.namespace, ) # Deploy the ECK all-in-one.yaml file self._deploy_eck() # Deploy the Elastic-Search server self._deploy_es() # Verify that ES is Up & Running sample = TimeoutSampler(timeout=180, sleep=10, func=self.get_health) if not sample.wait_for_func_status(True): raise Exception("Elasticsearch deployment Failed") # Deploy the elasticsearch dumper pod self._deploy_data_dumper_client() # Connect to the server self.con = self._es_connect()
def create_couchbase_worker(self, replicas=1): """ Deploy a Couchbase server and pillowfight workload using operator The couchbase workers do not come up unless there is an admission controller running. The admission controller is started from the default project prior to bringing up the operator. Secrets, rolebindings and serviceaccounts need to also be generated. Once the couchbase operator is running, we need to wait for the three worker pods to also be up. Then a pillowfight task is started. After the pillowfight task has finished, the log is collected and analyzed. Raises: Exception: If pillowfight results indicate that a minimum performance level is not reached (1 second response time, less than 1000 ops per second) """ logging.info("Creating pods..") cb_example = templating.load_yaml(constants.COUCHBASE_WORKER_EXAMPLE) if storagecluster_independent_check(): cb_example["spec"]["volumeClaimTemplates"][0]["spec"][ "storageClassName"] = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_RBD cb_example["spec"]["servers"][0]["size"] = replicas self.cb_examples = OCS(**cb_example) self.cb_examples.create() # Wait for last of three workers to be running. logging.info("Waiting for the pods to Running") for cb_wrk_pods in TimeoutSampler( self.WAIT_FOR_TIME, 3, get_pod_name_by_pattern, "cb-example", constants.COUCHBASE_OPERATOR, ): try: if len(cb_wrk_pods) == replicas: counter = 0 for cb_pod in cb_wrk_pods: if self.is_up_and_running(cb_pod, self.up_check): counter += 1 logging.info(f"Couchbase worker {cb_pod} is up") if counter == replicas: break except IndexError: logging.info( f"Expected number of couchbase pods are {replicas} " f"but only found {len(cb_wrk_pods)}")
def _wait_for_pv_backingstore_resource_deleted(namespace=None): """ wait for pv backing store resources to be deleted at the end of test teardown Args: backingstore_name (str): backingstore name namespace (str): backing store's namespace """ namespace = namespace or config.ENV_DATA["cluster_namespace"] sample = TimeoutSampler( timeout=120, sleep=15, func=_check_resources_deleted, namespace=namespace, ) if not sample.wait_for_func_status(result=True): log.error( f"{self.name} was not deleted properly, leftovers were found" ) raise TimeoutExpiredError
def check_backingstore_state( self, backingstore_name, desired_state, timeout=600 ): """ Checks whether the backing store reached a specific state Args: backingstore_name (str): Name of the backing store to be checked desired_state (str): The desired state of the backing store timeout (int): Number of seconds for timeout which will be used in the checks used in this function. Returns: bool: Whether the backing store has reached the desired state """ def _check_state(): sysinfo = self.send_rpc_query( 'system_api', 'read_system', params={} ).json()['reply'] for pool in sysinfo.get('pools'): if pool.get('name') == backingstore_name: current_state = pool.get('mode') logger.info( f'Current state of backingstore ''{backingstore_name} ' f'is {current_state}' ) if current_state == desired_state: return True return False try: for reached_state in TimeoutSampler(timeout, 10, _check_state): if reached_state: logger.info( f'BackingStore {backingstore_name} reached state ' f'{desired_state}.' ) return True else: logger.info( f'Waiting for BackingStore {backingstore_name} to ' f'reach state {desired_state}...' ) except TimeoutExpiredError: logger.error( f'The BackingStore did not reach the desired state ' f'{desired_state} within the time limit.' ) assert False
def internal_delete_uls(self, name): """ Deletes the Underlying Storage using the S3 API Args: name (str): The Underlying Storage name to be deleted """ assert self.exec_uls_deletion(name) # Todo: rename client to resource (or find an alternative) sample = TimeoutSampler(timeout=60, sleep=6, func=self.verify_uls_exists, uls_name=name) if not sample.wait_for_func_status(result=False): logger.error( f'Deletion of Underlying Storage {name} timed out. Unable to delete {name}' ) raise TimeoutExpiredError else: logger.info(f'Underlying Storage {name} deleted successfully.')
def wait_for_pv_backingstore_resource_deleted(backingstore_name, namespace=None): """ wait for pv backing store resources to be deleted at the end of test teardown Args: backingstore_name (str): backingstore name namespace (str): backing store's namespace """ namespace = namespace or config.ENV_DATA["cluster_namespace"] sample = TimeoutSampler( timeout=120, sleep=15, func=check_resources_deleted, backingstore_name=backingstore_name, namespace=namespace, ) if not sample.wait_for_func_status(result=True): log.error(f"Unable to delete resources of {backingstore_name}") raise TimeoutExpiredError
def expand_and_verify(self, pvc_size_new): """ Modify size of PVC and verify the change Args: pvc_size_new (int): Size of PVC(in Gb) to expand """ for pvc_obj in self.pvcs_cephfs + self.pvcs_rbd: log.info( f"Expanding size of PVC {pvc_obj.name} to {pvc_size_new}G" ) pvc_obj.resize_pvc(pvc_size_new, True) log.info(f"Verified: Size of all PVCs are expanded to {pvc_size_new}G") log.info("Verifying new size on pods.") for pod_obj in self.pods: if pod_obj.pvc.volume_mode == 'Block': log.info( f"Skipping check on pod {pod_obj.name} as volume " f"mode is Block." ) continue # Wait for 240 seconds to reflect the change on pod log.info(f"Checking pod {pod_obj.name} to verify the change.") for df_out in TimeoutSampler( 240, 3, pod_obj.exec_cmd_on_pod, command='df -kh' ): df_out = df_out.split() new_size_mount = df_out[ df_out.index(pod_obj.get_storage_path()) - 4 ] if new_size_mount in [ f'{pvc_size_new - 0.1}G', f'{float(pvc_size_new)}G', f'{pvc_size_new}G' ]: log.info( f"Verified: Expanded size of PVC {pod_obj.pvc.name} " f"is reflected on pod {pod_obj.name}" ) break log.info( f"Expanded size of PVC {pod_obj.pvc.name} is not reflected" f" on pod {pod_obj.name}. New size on mount is not " f"{pvc_size_new}G as expected, but {new_size_mount}. " f"Checking again." ) log.info( f"Verified: Modified size {pvc_size_new}G is reflected " f"on all pods." )
def wait_for_state(self, state, timeout=480, sleep=5): """ Wait till state of catalog source resource is the same as required one passed in the state parameter. Args: state (str): Desired state of catalog source object timeout (int): Timeout in seconds to wait for desired state sleep (int): Time in seconds to sleep between attempts Raises: ResourceWrongStatusException: In case the catalog source is not in expected state. """ self.check_name_is_specified() sampler = TimeoutSampler(timeout, sleep, self.check_state, state=state) if not sampler.wait_for_func_status(True): raise ResourceWrongStatusException( f"Catalog source: {self.resource_name} is not in expected " f"state: {state}")
def resize_pvc(self, pvc_size_new): """ Resize PVC Args: pvc_size_new (int): new pvc size """ for pvc_obj in self.pvc_objs: logger.info( f"Resize pvc {pvc_obj.name} sc_name={pvc_obj.storageclass.name}, " f"resize from {pvc_obj.size} to {pvc_size_new}, access_mode=" f"{pvc_obj.access_mode},volume_mode={pvc_obj.get_pvc_vol_mode}" ) pvc_obj.resize_pvc(new_size=pvc_size_new, verify=True) logger.info( f"Verified: Size of all PVCs are expanded to {pvc_size_new}G") logger.info("Verifying new size on pods.") for pod_obj in self.pod_objs: if pod_obj.pvc.get_pvc_vol_mode == "Block": logger.info(f"Skipping check on pod {pod_obj.name} as volume " f"mode is Block.") continue # Wait for 240 seconds to reflect the change on pod logger.info(f"Checking pod {pod_obj.name} to verify the change.") for df_out in TimeoutSampler(240, 3, pod_obj.exec_cmd_on_pod, command="df -kh"): if not df_out: continue df_out = df_out.split() new_size_mount = df_out[ df_out.index(pod_obj.get_storage_path()) - 4] if new_size_mount in [ f"{pvc_size_new - 0.1}G", f"{float(pvc_size_new)}G", f"{pvc_size_new}G", ]: logger.info( f"Verified: Expanded size of PVC {pod_obj.pvc.name} " f"is reflected on pod {pod_obj.name}") break logger.info( f"Expanded size of PVC {pod_obj.pvc.name} is not reflected" f" on pod {pod_obj.name}. New size on mount is not " f"{pvc_size_new}G as expected, but {new_size_mount}. " f"Checking again.") logger.info( f"Verified: Modified size {pvc_size_new}G is reflected on all pods." )
def test_ts_one_iteration(): """ When timeout == sleep_time, one iteration should happen. """ timeout = 1 sleep_time = 1 func = lambda: 1 # noqa: E731 results = [] with pytest.raises(TimeoutExpiredError): for result in TimeoutSampler(timeout, sleep_time, func): results.append(result) assert results == [1]
def wait_for_cache(mcg_obj, bucket_name, expected_objects_names=None): """ wait for existing cache bucket to cache all required objects Args: mcg_obj (MCG): An MCG object containing the MCG S3 connection credentials bucket_name (str): Name of the cache bucket expected_objects_names (list): Expected objects to be cached """ sample = TimeoutSampler( timeout=60, sleep=10, func=check_cached_objects_by_name, mcg_obj=mcg_obj, bucket_name=bucket_name, expected_objects_names=expected_objects_names, ) if not sample.wait_for_func_status(result=True): logger.error("Objects were not able to cache properly") raise UnexpectedBehaviour
def detach_and_delete_vols(self, volumes): """ Detach and delete volumes from the list Args: volumes (list): of Volume objects """ for v in volumes: if v.status == "in-use": v.detach() v.get() sample = TimeoutSampler( 100, 5, self.check_expected_vol_status, vol=v, expected_state="available", ) if not sample.wait_for_func_status(True): logger.error(f"Volume {v.name} failed to detach") raise exceptions.PSIVolumeNotInExpectedState() v.delete() sample = TimeoutSampler(100, 5, self.check_vol_deleted, vol=v) if not sample.wait_for_func_status(True): logger.error(f"Failed to delete Volume {v.name}") raise exceptions.PSIVolumeDeletionFailed()
def finalizer(): # Use provider cluster in managed service platform if self.consumer_cluster_index is not None: config.switch_to_provider() # Validate all mon services are running if len(mon_svc_list) != len( get_services_by_label( label=constants.MON_APP_LABEL, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE, )): # Restart the rook-operator pod operator_pod_obj = get_operator_pods() delete_pods(pod_objs=operator_pod_obj) POD_OBJ.wait_for_resource( condition=constants.STATUS_RUNNING, selector=constants.OPERATOR_LABEL, ) # Wait till all mon services are up for svc_list in TimeoutSampler( 1200, len(mon_svc_list), get_services_by_label, constants.MON_APP_LABEL, constants.OPENSHIFT_STORAGE_NAMESPACE, ): try: if len(svc_list) == len(mon_svc_list): log.info("All expected mon services are up") break except IndexError: log.error( f"All expected mon services are not up only found :{svc_list}. " f"Expected: {mon_svc_list}") # Wait till all mon pods running POD_OBJ.wait_for_resource( condition=constants.STATUS_RUNNING, selector=constants.MON_APP_LABEL, resource_count=len(mon_pods_list), timeout=600, sleep=3, ) # Check the ceph health OK ceph_health_check(tries=90, delay=15) # Switch the context to consumer cluster if needed if self.consumer_cluster_index is not None: config.switch_to_consumer(self.consumer_cluster_index)
def wait_for_pv_backingstore(backingstore_name, namespace=None): """ wait for existing pv backing store to reach OPTIMAL state Args: backingstore_name (str): backingstore name namespace (str): backing store's namespace """ namespace = namespace or config.ENV_DATA['cluster_namespace'] sample = TimeoutSampler(timeout=240, sleep=15, func=check_pv_backingstore_status, backingstore_name=backingstore_name, namespace=namespace) if not sample.wait_for_func_status(result=True): log.error( f'Backing Store {backingstore_name} never reached OPTIMAL state') raise TimeoutExpiredError else: log.info(f'Backing Store {backingstore_name} created successfully')
def wait_for_bg_operations(self, bg_ops, timeout=1200): """ Waits for threads to be completed Args: bg_ops (list): Futures timeout (int): Time in seconds to wait """ self.OPERATION_COMPLETED = True for thread in bg_ops: sample = TimeoutSampler(timeout=timeout, sleep=10, func=thread.done) assert sample.wait_for_func_status(result=True) try: logger.info(f"Thread completed: {thread.result()}") except exceptions.CommandFailed: logger.exception("Thread failed to complete") raise except Exception: logger.exception("Found an exception") raise
def delete_fio_data(fio_job_file, delete_check_func): """ Delete fio data by removing the fio job resource, with a wait to make sure date were reclaimed on the ceph level. """ # make sure we communicate what is going to happen logger.info(f"going to delete {fio_job_file.name} Job") fio_job_file.delete() logger.info(f"going to wait a bit to make sure that " f"data written by {fio_job_file.name} Job are really deleted") check_timeout = 660 # seconds check_sampler = TimeoutSampler(timeout=check_timeout, sleep=30, func=delete_check_func) finished_in_time = check_sampler.wait_for_func_status(result=True) if not finished_in_time: error_msg = ("it seems that the storage space was not reclaimed " f"within {check_timeout} seconds, " "this is most likely a product bug or misconfiguration") logger.error(error_msg) raise Exception(error_msg)
def deploy(self, log_level=""): self.flexy_instance.deploy(log_level) self.test_cluster() # add disks to instances # Get all instances and for each instance add # one disk pattern = "-".join( [get_infra_id(config.ENV_DATA["cluster_path"]), "compute"]) for instance in self.utils.get_instances_with_pattern(pattern): vol = self.utils.create_volume( name=f"{pattern}-disk0-{instance.name[-1]}", size=config.FLEXY["volume_size"], ) # wait till volume is available sample = TimeoutSampler(300, 10, self.utils.check_expected_vol_status, vol, "available") if not sample.wait_for_func_status(True): logger.info("Volume failed to reach 'available'") raise exceptions.PSIVolumeNotInExpectedState # attach the volume self.utils.attach_volume(vol, instance.id)
def wait_for_pods_and_verify(self): """ Wait for both the pods to be created and verify only one pod is running """ # Wait for pods for pods in TimeoutSampler(180, 2, func=pod.get_all_pods, namespace=self.namespace, selector=[self.name], selector_label='name'): if len(pods) == 2: break pods_iter = cycle(pods) # Wait for one pod to be in Running state sampler = TimeoutSampler(180, 2, next(pods_iter).get) for pod_info in sampler: sampler.func = next(pods_iter).get if pod_info['status']['phase'] == constants.STATUS_RUNNING: self.running_pod = next(pods_iter) break # Verify the other pod is not coming up Running try: self.pod_not_running = next(pods_iter) wait_for_resource_state(resource=self.pod_not_running, state=constants.STATUS_RUNNING, timeout=10) # ResourceWrongStatusException exception should be raised in # wait_for_resource_state. If not, raise UnexpectedBehaviour. raise UnexpectedBehaviour( f"Unexpected: Pod {self.pod_not_running.name} is in " f"Running state") except ResourceWrongStatusException: log.info(f"Verified: Only one pod {self.running_pod.name} is in " f"running state.")
def internal_delete_uls(self, name): """ Deletes the Underlying Storage using the S3 API Args: name (str): The Underlying Storage name to be deleted """ assert self.exec_uls_deletion(name) # Todo: rename client to resource (or find an alternative) sample = TimeoutSampler(timeout=180, sleep=15, func=self.verify_uls_exists, uls_name=name) if not sample.wait_for_func_status(result=False): logger.error( f'Deletion of Underlying Storage {name} timed out. Unable to delete {name}' ) logger.warning( f'AWS S3 bucket {name} still found after 3 minutes, and might require manual removal.' ) else: logger.info(f'Underlying Storage {name} deleted successfully.')
def test_pvc_expand_expanded_pvc(self): """ Verify PVC expand of already expanded PVC """ pvc_size_expanded_1 = 20 pvc_size_expanded_2 = 25 executor = ThreadPoolExecutor(max_workers=len(self.pods)) # Do setup on pods for running IO log.info("Setting up pods for running IO.") for pod_obj in self.pods: log.info(f"Setting up pod {pod_obj.name} for running IO") if pod_obj.pvc.volume_mode == 'Block': storage_type = 'block' else: storage_type = 'fs' executor.submit(pod_obj.workload_setup, storage_type=storage_type) # Wait for setup on pods to complete for pod_obj in self.pods: log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}") for sample in TimeoutSampler(360, 2, getattr, pod_obj, 'wl_setup_done'): if sample: log.info(f"Setup for running IO is completed on pod " f"{pod_obj.name}.") break log.info("Setup for running IO is completed on all pods.") # Run IO and verify log.info("Starting pre-expand IO on all pods.") self.run_io_and_verify(9, 'pre_expand') log.info("Verified pre-expand IO result on pods.") log.info("Expanding all PVCs.") self.expand_and_verify(pvc_size_expanded_1) # Run IO and verify log.info("Starting post-expand IO on all pods.") self.run_io_and_verify(8, 'post_expand') log.info("Verified post-expand IO result on pods.") log.info("Expanding all PVCs for the second time.") self.expand_and_verify(pvc_size_expanded_2) # Run IO and verify log.info("Starting post-second-expand IO on all pods.") self.run_io_and_verify(6, 'post_expand') log.info("Verified post-second-expand IO result on pods.")
def _deploy_eck(self): """ Deploying the ECK environment for the Elasticsearch, and make sure it is in Running mode """ log.info("Deploying the ECK environment for the ES cluster") log.info("Deploy the ECK CRD's") self.ocp.apply(self.eck_file) log.info("deploy the ECK operator") self.ocp.apply(f"{self.eck_path}/operator.yaml") sample = TimeoutSampler(timeout=300, sleep=10, func=self._pod_is_found, pattern="elastic-operator") if not sample.wait_for_func_status(True): err_msg = "ECK deployment Failed" log.error(err_msg) self.cleanup() raise Exception(err_msg) log.info("The ECK pod is ready !")
def wait_for_resource(self, condition, resource_name='', selector=None, resource_count=0, timeout=60, sleep=3): """ Wait for a resource to reach to a desired condition Args: condition (str): The desired state the resource should be at This is referring to: status.phase presented in the resource yaml file (e.g. status.phase == Available) resource_name (str): The name of the resource to wait for (e.g.my-pv1) selector (str): The resource selector to search with. Example: 'app=rook-ceph-mds' resource_count (int): How many resources expected to be timeout (int): Time in seconds to wait sleep (int): Sampling time in seconds Returns: bool: True in case all resources reached desired condition, False otherwise """ for sample in TimeoutSampler(timeout, sleep, self.get, resource_name, True, selector): # Only 1 resource expected to be returned if resource_name: if sample.get('status').get('phase') == condition: return True # More than 1 resources returned elif sample.get('kind') == 'List': in_condition = [] sample = sample['items'] for item in sample: if item.get('status').get('phase') == condition: in_condition.append(item) if resource_count: if len(in_condition) == resource_count and ( len(sample) == len(in_condition)): return True elif len(sample) == len(in_condition): return True return False
def wait_for_phase(self, phase, timeout=300, sleep=5): """ Wait till phase of CSV resource is the same as required one passed in the phase parameter. Args: phase (str): Desired phase of CSV object timeout (int): Timeout in seconds to wait for desired phase sleep (int): Time in seconds to sleep between attempts Raises: ResourceInUnexpectedState: In case the CSV is not in expected phase. """ self.check_name_is_specified() sampler = TimeoutSampler( timeout, sleep, self.check_phase, phase=phase ) if not sampler.wait_for_func_status(True): raise ResourceInUnexpectedState( f"CSV: {self.resource_name} is not in expected phase: {phase}" )
def wait_for_vr_count(count, namespace, timeout=300): """ Wait for all VR resources to reach expected count in the given namespace Args: count (int): Expected number of VR resources namespace (str): the namespace of the VR resources timeout (int): time in seconds to wait for VR resources to be created or reach expected state Returns: bool: True if all VR are in expected state """ sample = TimeoutSampler( timeout=timeout, sleep=5, func=get_vr_count, namespace=namespace, ) sample.wait_for_func_value(count) return True
def internal_delete_uls(self, name): """ Deletes the Underlying Storage using the S3 API Args: name (str): The Underlying Storage name to be deleted """ def _exec_uls_deletion(self, name): """ Try to delete Underlying Storage by name if exists Args: name (str): the Underlying Storage name Returns: bool: True if deleted successfully """ if self.verify_uls_exists(name): try: # TODO: Check why bucket policy deletion fails on IBM COS # when bucket have no policy set if "aws" in name: self.client.meta.client.delete_bucket_policy( Bucket=name) self.client.Bucket(name).objects.all().delete() self.client.Bucket(name).delete() return True except ClientError: logger.info( f"Deletion of Underlying Storage {name} failed.") return False else: logger.warning( f"Underlying Storage {name} does not exist, and was not deleted." ) return True try: for deletion_result in TimeoutSampler(60, 5, _exec_uls_deletion, self, name): if deletion_result: logger.info("ULS deleted.") break except TimeoutExpiredError: logger.error("Failed to delete ULS.") assert False
def test_sql_workload_simple(self, ripsaw): """ This is a basic pgsql workload """ # Deployment postgres log.info("Deploying postgres database") ripsaw.apply_crd('resources/crds/' 'ripsaw_v1alpha1_ripsaw_crd.yaml') ripsaw.setup_postgresql() # Create pgbench benchmark log.info("Create resource file for pgbench workload") pg_data = templating.load_yaml(constants.PGSQL_BENCHMARK_YAML) pg_obj = OCS(**pg_data) pg_obj.create() # Wait for pgbench pod to be created for pgbench_pod in TimeoutSampler(300, 3, get_pod_name_by_pattern, 'pgbench-1-dbs-client', 'my-ripsaw'): try: if pgbench_pod[0] is not None: pgbench_client_pod = pgbench_pod[0] break except IndexError: log.info("Bench pod not ready yet") # Wait for pg_bench pod to initialized and complete log.info("Waiting for pgbench_client to complete") pod_obj = OCP(kind='pod') pod_obj.wait_for_resource( condition='Completed', resource_name=pgbench_client_pod, timeout=800, sleep=10, ) # Running pgbench and parsing logs output = run_cmd(f'oc logs {pgbench_client_pod}') pg_output = utils.parse_pgsql_logs(output) log.info("*******PGBench output log*********\n" f"{pg_output}") for data in pg_output: latency_avg = data['latency_avg'] if not latency_avg: raise UnexpectedBehaviour("PGBench failed to run, " "no data found on latency_avg") log.info("PGBench has completed successfully") # Clean up pgbench benchmark log.info("Deleting PG bench benchmark") pg_obj.delete()
def verify_operator_succeeded( self, operator="OpenShift Container Storage", timeout_install=300, sleep=20 ): """ Verify Operator Installation Args: operator (str): type of operator timeout_install (int): Time in seconds to wait sleep (int): Sampling time in seconds """ self.search_operator_installed_operators_page(operator=operator) sample = TimeoutSampler( timeout=timeout_install, sleep=sleep, func=self.check_element_text, expected_text="Succeeded", ) if not sample.wait_for_func_status(result=True): logger.error( f"{operator} Installation status is not Succeeded after {timeout_install} seconds" ) raise TimeoutExpiredError
def test_fio_with_block_storage(self): name = 'test_workload' spec = self.pod_obj.data.get('spec') path = ( spec.get('containers')[0].get('volumeMounts')[0].get('mountPath') ) work_load = 'fio' storage_type = 'fs' # few io parameters for Fio runtime = 10 size = '200M' wl = workload.WorkLoad( name, path, work_load, storage_type, self.pod_obj ) assert wl.setup() io_params = templating.load_yaml_to_dict(constants.FIO_IO_PARAMS_YAML) io_params['runtime'] = runtime io_params['size'] = size future_result = wl.run(**io_params) timeout = 1200 sample = TimeoutSampler( timeout=timeout, sleep=3, func=future_result.done ) assert sample.wait_for_func_status(result=True) try: logger.info(future_result.result()) except exceptions.CommandFailed: logger.exception(f"FIO failed") raise except Exception: logger.exception(f"Found Exception") raise
def reboot_rhv_vms(self, vms, timeout=600, wait=True, force=True): """ Reboot the RHV virtual machines Args: vms (list): Names of RHV vms timeout (int): time in seconds to wait for VM to reboot wait (bool): Wait for RHV VMs to reboot force (bool): True to reboot vm forcibly False otherwise """ for vm in vms: # Find the virtual machine vm_service = self.get_vm_service(vm.id) vm_service.reboot(force=force) if wait: # Wait till the virtual machine will start rebooting and then UP try: expc_status = types.VmStatus.REBOOT_IN_PROGRESS look_up = False for status in TimeoutSampler(timeout, 5, self.get_vm_status, vm): if not look_up and status != expc_status: logger.info( f"Waiting for RHV Machine {vm.name} to {expc_status}" f" Current status is : {status}") continue elif not look_up and status == expc_status: expc_status = types.VmStatus.UP look_up = True logger.info( f"RHV Machine {vm} is now {expc_status} status" ) continue else: logger.info( f"Waiting for RHV Machine {vm.name} to Power ON" f" Current status is : {status}") if status == types.VmStatus.UP: logger.info( f"RHV Machine {vm.name} reached UP status") break except TimeoutExpiredError: logger.error( f"RHV VM {vm.name} is still not {expc_status} after " f"initiating reboot") raise