def wait_for_nodes_status(node_names=None,
status=constants.NODE_READY,
timeout=180):
"""
Wait until all nodes are in the given status
Args:
node_names (list): The node names to wait for to reached the desired state
If None, will wait for all cluster nodes
status (str): The node status to wait for
(e.g. 'Ready', 'NotReady', 'SchedulingDisabled')
timeout (int): The number in seconds to wait for the nodes to reach
the status
Raises:
ResourceWrongStatusException: In case one or more nodes haven't
reached the desired state
"""
try:
if not node_names:
for sample in TimeoutSampler(60, 3, get_node_objs):
if sample:
node_names = [node.name for node in sample]
break
nodes_not_in_state = copy.deepcopy(node_names)
log.info(f"Waiting for nodes {node_names} to reach status {status}")
for sample in TimeoutSampler(timeout, 3, get_node_objs,
nodes_not_in_state):
for node in sample:
if node.ocp.get_resource_status(node.name) == status:
nodes_not_in_state.remove(node.name)
if not nodes_not_in_state:
break
except TimeoutExpiredError:
log.error(
f"The following nodes haven't reached status {status}: {node_names}"
)
raise exceptions.ResourceWrongStatusException(
node_names, [n.describe() for n in get_node_objs(node_names)])
def __init__(self):
"""
Initializer function
"""
log.info("Initializing the Elastic-Search environment object")
self.namespace = "elastic-system"
self.eck_file = "ocs_ci/templates/app-pods/eck.1.6.0-all-in-one.yaml"
self.dumper_file = "ocs_ci/templates/app-pods/esclient.yaml"
self.crd = "ocs_ci/templates/app-pods/esq.yaml"
# Creating some different types of OCP objects
self.ocp = OCP(kind="pod",
resource_name="elastic-operator-0",
namespace=self.namespace)
self.ns_obj = OCP(kind="namespace", namespace=self.namespace)
self.es = OCP(resource_name="quickstart-es-http",
namespace=self.namespace)
self.elasticsearch = OCP(namespace=self.namespace,
kind="elasticsearch")
self.password = OCP(
kind="secret",
resource_name="quickstart-es-elastic-user",
namespace=self.namespace,
)
# Deploy the ECK all-in-one.yaml file
self._deploy_eck()
# Deploy the Elastic-Search server
self._deploy_es()
# Verify that ES is Up & Running
sample = TimeoutSampler(timeout=180, sleep=10, func=self.get_health)
if not sample.wait_for_func_status(True):
raise Exception("Elasticsearch deployment Failed")
# Deploy the elasticsearch dumper pod
self._deploy_data_dumper_client()
# Connect to the server
self.con = self._es_connect()
def create_couchbase_worker(self, replicas=1):
"""
Deploy a Couchbase server and pillowfight workload using operator
The couchbase workers do not come up unless there is an admission controller
running. The admission controller is started from the default project prior
to bringing up the operator. Secrets, rolebindings and serviceaccounts
need to also be generated.
Once the couchbase operator is running, we need to wait for the three
worker pods to also be up. Then a pillowfight task is started.
After the pillowfight task has finished, the log is collected and
analyzed.
Raises:
Exception: If pillowfight results indicate that a minimum performance
level is not reached (1 second response time, less than 1000 ops
per second)
"""
logging.info("Creating pods..")
cb_example = templating.load_yaml(constants.COUCHBASE_WORKER_EXAMPLE)
if storagecluster_independent_check():
cb_example["spec"]["volumeClaimTemplates"][0]["spec"][
"storageClassName"] = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_RBD
cb_example["spec"]["servers"][0]["size"] = replicas
self.cb_examples = OCS(**cb_example)
self.cb_examples.create()
# Wait for last of three workers to be running.
logging.info("Waiting for the pods to Running")
for cb_wrk_pods in TimeoutSampler(
self.WAIT_FOR_TIME,
3,
get_pod_name_by_pattern,
"cb-example",
constants.COUCHBASE_OPERATOR,
):
try:
if len(cb_wrk_pods) == replicas:
counter = 0
for cb_pod in cb_wrk_pods:
if self.is_up_and_running(cb_pod, self.up_check):
counter += 1
logging.info(f"Couchbase worker {cb_pod} is up")
if counter == replicas:
break
except IndexError:
logging.info(
f"Expected number of couchbase pods are {replicas} "
f"but only found {len(cb_wrk_pods)}")
def _wait_for_pv_backingstore_resource_deleted(namespace=None):
"""
wait for pv backing store resources to be deleted at the end of test teardown
Args:
backingstore_name (str): backingstore name
namespace (str): backing store's namespace
"""
namespace = namespace or config.ENV_DATA["cluster_namespace"]
sample = TimeoutSampler(
timeout=120,
sleep=15,
func=_check_resources_deleted,
namespace=namespace,
)
if not sample.wait_for_func_status(result=True):
log.error(
f"{self.name} was not deleted properly, leftovers were found"
)
raise TimeoutExpiredError
def check_backingstore_state(
self,
backingstore_name,
desired_state,
timeout=600
):
"""
Checks whether the backing store reached a specific state
Args:
backingstore_name (str): Name of the backing store to be checked
desired_state (str): The desired state of the backing store
timeout (int): Number of seconds for timeout which will be used
in the checks used in this function.
Returns:
bool: Whether the backing store has reached the desired state
"""
def _check_state():
sysinfo = self.send_rpc_query(
'system_api', 'read_system', params={}
).json()['reply']
for pool in sysinfo.get('pools'):
if pool.get('name') == backingstore_name:
current_state = pool.get('mode')
logger.info(
f'Current state of backingstore ''{backingstore_name} '
f'is {current_state}'
)
if current_state == desired_state:
return True
return False
try:
for reached_state in TimeoutSampler(timeout, 10, _check_state):
if reached_state:
logger.info(
f'BackingStore {backingstore_name} reached state '
f'{desired_state}.'
)
return True
else:
logger.info(
f'Waiting for BackingStore {backingstore_name} to '
f'reach state {desired_state}...'
)
except TimeoutExpiredError:
logger.error(
f'The BackingStore did not reach the desired state '
f'{desired_state} within the time limit.'
)
assert False
def internal_delete_uls(self, name):
"""
Deletes the Underlying Storage using the S3 API
Args:
name (str): The Underlying Storage name to be deleted
"""
assert self.exec_uls_deletion(name)
# Todo: rename client to resource (or find an alternative)
sample = TimeoutSampler(timeout=60,
sleep=6,
func=self.verify_uls_exists,
uls_name=name)
if not sample.wait_for_func_status(result=False):
logger.error(
f'Deletion of Underlying Storage {name} timed out. Unable to delete {name}'
)
raise TimeoutExpiredError
else:
logger.info(f'Underlying Storage {name} deleted successfully.')
def wait_for_pv_backingstore_resource_deleted(backingstore_name,
namespace=None):
"""
wait for pv backing store resources to be deleted at the end of test teardown
Args:
backingstore_name (str): backingstore name
namespace (str): backing store's namespace
"""
namespace = namespace or config.ENV_DATA["cluster_namespace"]
sample = TimeoutSampler(
timeout=120,
sleep=15,
func=check_resources_deleted,
backingstore_name=backingstore_name,
namespace=namespace,
)
if not sample.wait_for_func_status(result=True):
log.error(f"Unable to delete resources of {backingstore_name}")
raise TimeoutExpiredError
def expand_and_verify(self, pvc_size_new):
"""
Modify size of PVC and verify the change
Args:
pvc_size_new (int): Size of PVC(in Gb) to expand
"""
for pvc_obj in self.pvcs_cephfs + self.pvcs_rbd:
log.info(
f"Expanding size of PVC {pvc_obj.name} to {pvc_size_new}G"
)
pvc_obj.resize_pvc(pvc_size_new, True)
log.info(f"Verified: Size of all PVCs are expanded to {pvc_size_new}G")
log.info("Verifying new size on pods.")
for pod_obj in self.pods:
if pod_obj.pvc.volume_mode == 'Block':
log.info(
f"Skipping check on pod {pod_obj.name} as volume "
f"mode is Block."
)
continue
# Wait for 240 seconds to reflect the change on pod
log.info(f"Checking pod {pod_obj.name} to verify the change.")
for df_out in TimeoutSampler(
240, 3, pod_obj.exec_cmd_on_pod, command='df -kh'
):
df_out = df_out.split()
new_size_mount = df_out[
df_out.index(pod_obj.get_storage_path()) - 4
]
if new_size_mount in [
f'{pvc_size_new - 0.1}G', f'{float(pvc_size_new)}G',
f'{pvc_size_new}G'
]:
log.info(
f"Verified: Expanded size of PVC {pod_obj.pvc.name} "
f"is reflected on pod {pod_obj.name}"
)
break
log.info(
f"Expanded size of PVC {pod_obj.pvc.name} is not reflected"
f" on pod {pod_obj.name}. New size on mount is not "
f"{pvc_size_new}G as expected, but {new_size_mount}. "
f"Checking again."
)
log.info(
f"Verified: Modified size {pvc_size_new}G is reflected "
f"on all pods."
)
def wait_for_state(self, state, timeout=480, sleep=5):
"""
Wait till state of catalog source resource is the same as required one
passed in the state parameter.
Args:
state (str): Desired state of catalog source object
timeout (int): Timeout in seconds to wait for desired state
sleep (int): Time in seconds to sleep between attempts
Raises:
ResourceWrongStatusException: In case the catalog source is not in
expected state.
"""
self.check_name_is_specified()
sampler = TimeoutSampler(timeout, sleep, self.check_state, state=state)
if not sampler.wait_for_func_status(True):
raise ResourceWrongStatusException(
f"Catalog source: {self.resource_name} is not in expected "
f"state: {state}")
def resize_pvc(self, pvc_size_new):
"""
Resize PVC
Args:
pvc_size_new (int): new pvc size
"""
for pvc_obj in self.pvc_objs:
logger.info(
f"Resize pvc {pvc_obj.name} sc_name={pvc_obj.storageclass.name}, "
f"resize from {pvc_obj.size} to {pvc_size_new}, access_mode="
f"{pvc_obj.access_mode},volume_mode={pvc_obj.get_pvc_vol_mode}"
)
pvc_obj.resize_pvc(new_size=pvc_size_new, verify=True)
logger.info(
f"Verified: Size of all PVCs are expanded to {pvc_size_new}G")
logger.info("Verifying new size on pods.")
for pod_obj in self.pod_objs:
if pod_obj.pvc.get_pvc_vol_mode == "Block":
logger.info(f"Skipping check on pod {pod_obj.name} as volume "
f"mode is Block.")
continue
# Wait for 240 seconds to reflect the change on pod
logger.info(f"Checking pod {pod_obj.name} to verify the change.")
for df_out in TimeoutSampler(240,
3,
pod_obj.exec_cmd_on_pod,
command="df -kh"):
if not df_out:
continue
df_out = df_out.split()
new_size_mount = df_out[
df_out.index(pod_obj.get_storage_path()) - 4]
if new_size_mount in [
f"{pvc_size_new - 0.1}G",
f"{float(pvc_size_new)}G",
f"{pvc_size_new}G",
]:
logger.info(
f"Verified: Expanded size of PVC {pod_obj.pvc.name} "
f"is reflected on pod {pod_obj.name}")
break
logger.info(
f"Expanded size of PVC {pod_obj.pvc.name} is not reflected"
f" on pod {pod_obj.name}. New size on mount is not "
f"{pvc_size_new}G as expected, but {new_size_mount}. "
f"Checking again.")
logger.info(
f"Verified: Modified size {pvc_size_new}G is reflected on all pods."
)
def test_ts_one_iteration():
"""
When timeout == sleep_time, one iteration should happen.
"""
timeout = 1
sleep_time = 1
func = lambda: 1 # noqa: E731
results = []
with pytest.raises(TimeoutExpiredError):
for result in TimeoutSampler(timeout, sleep_time, func):
results.append(result)
assert results == [1]
def wait_for_cache(mcg_obj, bucket_name, expected_objects_names=None):
"""
wait for existing cache bucket to cache all required objects
Args:
mcg_obj (MCG): An MCG object containing the MCG S3 connection credentials
bucket_name (str): Name of the cache bucket
expected_objects_names (list): Expected objects to be cached
"""
sample = TimeoutSampler(
timeout=60,
sleep=10,
func=check_cached_objects_by_name,
mcg_obj=mcg_obj,
bucket_name=bucket_name,
expected_objects_names=expected_objects_names,
)
if not sample.wait_for_func_status(result=True):
logger.error("Objects were not able to cache properly")
raise UnexpectedBehaviour
def detach_and_delete_vols(self, volumes):
"""
Detach and delete volumes from the list
Args:
volumes (list): of Volume objects
"""
for v in volumes:
if v.status == "in-use":
v.detach()
v.get()
sample = TimeoutSampler(
100,
5,
self.check_expected_vol_status,
vol=v,
expected_state="available",
)
if not sample.wait_for_func_status(True):
logger.error(f"Volume {v.name} failed to detach")
raise exceptions.PSIVolumeNotInExpectedState()
v.delete()
sample = TimeoutSampler(100, 5, self.check_vol_deleted, vol=v)
if not sample.wait_for_func_status(True):
logger.error(f"Failed to delete Volume {v.name}")
raise exceptions.PSIVolumeDeletionFailed()
def finalizer():
# Use provider cluster in managed service platform
if self.consumer_cluster_index is not None:
config.switch_to_provider()
# Validate all mon services are running
if len(mon_svc_list) != len(
get_services_by_label(
label=constants.MON_APP_LABEL,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE,
)):
# Restart the rook-operator pod
operator_pod_obj = get_operator_pods()
delete_pods(pod_objs=operator_pod_obj)
POD_OBJ.wait_for_resource(
condition=constants.STATUS_RUNNING,
selector=constants.OPERATOR_LABEL,
)
# Wait till all mon services are up
for svc_list in TimeoutSampler(
1200,
len(mon_svc_list),
get_services_by_label,
constants.MON_APP_LABEL,
constants.OPENSHIFT_STORAGE_NAMESPACE,
):
try:
if len(svc_list) == len(mon_svc_list):
log.info("All expected mon services are up")
break
except IndexError:
log.error(
f"All expected mon services are not up only found :{svc_list}. "
f"Expected: {mon_svc_list}")
# Wait till all mon pods running
POD_OBJ.wait_for_resource(
condition=constants.STATUS_RUNNING,
selector=constants.MON_APP_LABEL,
resource_count=len(mon_pods_list),
timeout=600,
sleep=3,
)
# Check the ceph health OK
ceph_health_check(tries=90, delay=15)
# Switch the context to consumer cluster if needed
if self.consumer_cluster_index is not None:
config.switch_to_consumer(self.consumer_cluster_index)
def wait_for_pv_backingstore(backingstore_name, namespace=None):
"""
wait for existing pv backing store to reach OPTIMAL state
Args:
backingstore_name (str): backingstore name
namespace (str): backing store's namespace
"""
namespace = namespace or config.ENV_DATA['cluster_namespace']
sample = TimeoutSampler(timeout=240,
sleep=15,
func=check_pv_backingstore_status,
backingstore_name=backingstore_name,
namespace=namespace)
if not sample.wait_for_func_status(result=True):
log.error(
f'Backing Store {backingstore_name} never reached OPTIMAL state')
raise TimeoutExpiredError
else:
log.info(f'Backing Store {backingstore_name} created successfully')
def wait_for_bg_operations(self, bg_ops, timeout=1200):
"""
Waits for threads to be completed
Args:
bg_ops (list): Futures
timeout (int): Time in seconds to wait
"""
self.OPERATION_COMPLETED = True
for thread in bg_ops:
sample = TimeoutSampler(timeout=timeout, sleep=10, func=thread.done)
assert sample.wait_for_func_status(result=True)
try:
logger.info(f"Thread completed: {thread.result()}")
except exceptions.CommandFailed:
logger.exception("Thread failed to complete")
raise
except Exception:
logger.exception("Found an exception")
raise
def delete_fio_data(fio_job_file, delete_check_func):
"""
Delete fio data by removing the fio job resource, with a wait to
make sure date were reclaimed on the ceph level.
"""
# make sure we communicate what is going to happen
logger.info(f"going to delete {fio_job_file.name} Job")
fio_job_file.delete()
logger.info(f"going to wait a bit to make sure that "
f"data written by {fio_job_file.name} Job are really deleted")
check_timeout = 660 # seconds
check_sampler = TimeoutSampler(timeout=check_timeout,
sleep=30,
func=delete_check_func)
finished_in_time = check_sampler.wait_for_func_status(result=True)
if not finished_in_time:
error_msg = ("it seems that the storage space was not reclaimed "
f"within {check_timeout} seconds, "
"this is most likely a product bug or misconfiguration")
logger.error(error_msg)
raise Exception(error_msg)
def deploy(self, log_level=""):
self.flexy_instance.deploy(log_level)
self.test_cluster()
# add disks to instances
# Get all instances and for each instance add
# one disk
pattern = "-".join(
[get_infra_id(config.ENV_DATA["cluster_path"]), "compute"])
for instance in self.utils.get_instances_with_pattern(pattern):
vol = self.utils.create_volume(
name=f"{pattern}-disk0-{instance.name[-1]}",
size=config.FLEXY["volume_size"],
)
# wait till volume is available
sample = TimeoutSampler(300, 10,
self.utils.check_expected_vol_status,
vol, "available")
if not sample.wait_for_func_status(True):
logger.info("Volume failed to reach 'available'")
raise exceptions.PSIVolumeNotInExpectedState
# attach the volume
self.utils.attach_volume(vol, instance.id)
def wait_for_pods_and_verify(self):
"""
Wait for both the pods to be created and verify only one pod is running
"""
# Wait for pods
for pods in TimeoutSampler(180,
2,
func=pod.get_all_pods,
namespace=self.namespace,
selector=[self.name],
selector_label='name'):
if len(pods) == 2:
break
pods_iter = cycle(pods)
# Wait for one pod to be in Running state
sampler = TimeoutSampler(180, 2, next(pods_iter).get)
for pod_info in sampler:
sampler.func = next(pods_iter).get
if pod_info['status']['phase'] == constants.STATUS_RUNNING:
self.running_pod = next(pods_iter)
break
# Verify the other pod is not coming up Running
try:
self.pod_not_running = next(pods_iter)
wait_for_resource_state(resource=self.pod_not_running,
state=constants.STATUS_RUNNING,
timeout=10)
# ResourceWrongStatusException exception should be raised in
# wait_for_resource_state. If not, raise UnexpectedBehaviour.
raise UnexpectedBehaviour(
f"Unexpected: Pod {self.pod_not_running.name} is in "
f"Running state")
except ResourceWrongStatusException:
log.info(f"Verified: Only one pod {self.running_pod.name} is in "
f"running state.")
def internal_delete_uls(self, name):
"""
Deletes the Underlying Storage using the S3 API
Args:
name (str): The Underlying Storage name to be deleted
"""
assert self.exec_uls_deletion(name)
# Todo: rename client to resource (or find an alternative)
sample = TimeoutSampler(timeout=180,
sleep=15,
func=self.verify_uls_exists,
uls_name=name)
if not sample.wait_for_func_status(result=False):
logger.error(
f'Deletion of Underlying Storage {name} timed out. Unable to delete {name}'
)
logger.warning(
f'AWS S3 bucket {name} still found after 3 minutes, and might require manual removal.'
)
else:
logger.info(f'Underlying Storage {name} deleted successfully.')
def test_pvc_expand_expanded_pvc(self):
"""
Verify PVC expand of already expanded PVC
"""
pvc_size_expanded_1 = 20
pvc_size_expanded_2 = 25
executor = ThreadPoolExecutor(max_workers=len(self.pods))
# Do setup on pods for running IO
log.info("Setting up pods for running IO.")
for pod_obj in self.pods:
log.info(f"Setting up pod {pod_obj.name} for running IO")
if pod_obj.pvc.volume_mode == 'Block':
storage_type = 'block'
else:
storage_type = 'fs'
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on pods to complete
for pod_obj in self.pods:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(360, 2, getattr, pod_obj,
'wl_setup_done'):
if sample:
log.info(f"Setup for running IO is completed on pod "
f"{pod_obj.name}.")
break
log.info("Setup for running IO is completed on all pods.")
# Run IO and verify
log.info("Starting pre-expand IO on all pods.")
self.run_io_and_verify(9, 'pre_expand')
log.info("Verified pre-expand IO result on pods.")
log.info("Expanding all PVCs.")
self.expand_and_verify(pvc_size_expanded_1)
# Run IO and verify
log.info("Starting post-expand IO on all pods.")
self.run_io_and_verify(8, 'post_expand')
log.info("Verified post-expand IO result on pods.")
log.info("Expanding all PVCs for the second time.")
self.expand_and_verify(pvc_size_expanded_2)
# Run IO and verify
log.info("Starting post-second-expand IO on all pods.")
self.run_io_and_verify(6, 'post_expand')
log.info("Verified post-second-expand IO result on pods.")
def _deploy_eck(self):
"""
Deploying the ECK environment for the Elasticsearch, and make sure it
is in Running mode
"""
log.info("Deploying the ECK environment for the ES cluster")
log.info("Deploy the ECK CRD's")
self.ocp.apply(self.eck_file)
log.info("deploy the ECK operator")
self.ocp.apply(f"{self.eck_path}/operator.yaml")
sample = TimeoutSampler(timeout=300,
sleep=10,
func=self._pod_is_found,
pattern="elastic-operator")
if not sample.wait_for_func_status(True):
err_msg = "ECK deployment Failed"
log.error(err_msg)
self.cleanup()
raise Exception(err_msg)
log.info("The ECK pod is ready !")
def wait_for_resource(self,
condition,
resource_name='',
selector=None,
resource_count=0,
timeout=60,
sleep=3):
"""
Wait for a resource to reach to a desired condition
Args:
condition (str): The desired state the resource should be at
This is referring to: status.phase presented in the resource
yaml file
(e.g. status.phase == Available)
resource_name (str): The name of the resource to wait
for (e.g.my-pv1)
selector (str): The resource selector to search with.
Example: 'app=rook-ceph-mds'
resource_count (int): How many resources expected to be
timeout (int): Time in seconds to wait
sleep (int): Sampling time in seconds
Returns:
bool: True in case all resources reached desired condition,
False otherwise
"""
for sample in TimeoutSampler(timeout, sleep, self.get, resource_name,
True, selector):
# Only 1 resource expected to be returned
if resource_name:
if sample.get('status').get('phase') == condition:
return True
# More than 1 resources returned
elif sample.get('kind') == 'List':
in_condition = []
sample = sample['items']
for item in sample:
if item.get('status').get('phase') == condition:
in_condition.append(item)
if resource_count:
if len(in_condition) == resource_count and (
len(sample) == len(in_condition)):
return True
elif len(sample) == len(in_condition):
return True
return False
def wait_for_phase(self, phase, timeout=300, sleep=5):
"""
Wait till phase of CSV resource is the same as required one passed in
the phase parameter.
Args:
phase (str): Desired phase of CSV object
timeout (int): Timeout in seconds to wait for desired phase
sleep (int): Time in seconds to sleep between attempts
Raises:
ResourceInUnexpectedState: In case the CSV is not in expected
phase.
"""
self.check_name_is_specified()
sampler = TimeoutSampler(
timeout, sleep, self.check_phase, phase=phase
)
if not sampler.wait_for_func_status(True):
raise ResourceInUnexpectedState(
f"CSV: {self.resource_name} is not in expected phase: {phase}"
)
def wait_for_vr_count(count, namespace, timeout=300):
"""
Wait for all VR resources to reach expected count in the given namespace
Args:
count (int): Expected number of VR resources
namespace (str): the namespace of the VR resources
timeout (int): time in seconds to wait for VR resources to be created
or reach expected state
Returns:
bool: True if all VR are in expected state
"""
sample = TimeoutSampler(
timeout=timeout,
sleep=5,
func=get_vr_count,
namespace=namespace,
)
sample.wait_for_func_value(count)
return True
def internal_delete_uls(self, name):
"""
Deletes the Underlying Storage using the S3 API
Args:
name (str): The Underlying Storage name to be deleted
"""
def _exec_uls_deletion(self, name):
"""
Try to delete Underlying Storage by name if exists
Args:
name (str): the Underlying Storage name
Returns:
bool: True if deleted successfully
"""
if self.verify_uls_exists(name):
try:
# TODO: Check why bucket policy deletion fails on IBM COS
# when bucket have no policy set
if "aws" in name:
self.client.meta.client.delete_bucket_policy(
Bucket=name)
self.client.Bucket(name).objects.all().delete()
self.client.Bucket(name).delete()
return True
except ClientError:
logger.info(
f"Deletion of Underlying Storage {name} failed.")
return False
else:
logger.warning(
f"Underlying Storage {name} does not exist, and was not deleted."
)
return True
try:
for deletion_result in TimeoutSampler(60, 5, _exec_uls_deletion,
self, name):
if deletion_result:
logger.info("ULS deleted.")
break
except TimeoutExpiredError:
logger.error("Failed to delete ULS.")
assert False
def test_sql_workload_simple(self, ripsaw):
"""
This is a basic pgsql workload
"""
# Deployment postgres
log.info("Deploying postgres database")
ripsaw.apply_crd('resources/crds/' 'ripsaw_v1alpha1_ripsaw_crd.yaml')
ripsaw.setup_postgresql()
# Create pgbench benchmark
log.info("Create resource file for pgbench workload")
pg_data = templating.load_yaml(constants.PGSQL_BENCHMARK_YAML)
pg_obj = OCS(**pg_data)
pg_obj.create()
# Wait for pgbench pod to be created
for pgbench_pod in TimeoutSampler(300, 3, get_pod_name_by_pattern,
'pgbench-1-dbs-client', 'my-ripsaw'):
try:
if pgbench_pod[0] is not None:
pgbench_client_pod = pgbench_pod[0]
break
except IndexError:
log.info("Bench pod not ready yet")
# Wait for pg_bench pod to initialized and complete
log.info("Waiting for pgbench_client to complete")
pod_obj = OCP(kind='pod')
pod_obj.wait_for_resource(
condition='Completed',
resource_name=pgbench_client_pod,
timeout=800,
sleep=10,
)
# Running pgbench and parsing logs
output = run_cmd(f'oc logs {pgbench_client_pod}')
pg_output = utils.parse_pgsql_logs(output)
log.info("*******PGBench output log*********\n" f"{pg_output}")
for data in pg_output:
latency_avg = data['latency_avg']
if not latency_avg:
raise UnexpectedBehaviour("PGBench failed to run, "
"no data found on latency_avg")
log.info("PGBench has completed successfully")
# Clean up pgbench benchmark
log.info("Deleting PG bench benchmark")
pg_obj.delete()
def verify_operator_succeeded(
self, operator="OpenShift Container Storage", timeout_install=300, sleep=20
):
"""
Verify Operator Installation
Args:
operator (str): type of operator
timeout_install (int): Time in seconds to wait
sleep (int): Sampling time in seconds
"""
self.search_operator_installed_operators_page(operator=operator)
sample = TimeoutSampler(
timeout=timeout_install,
sleep=sleep,
func=self.check_element_text,
expected_text="Succeeded",
)
if not sample.wait_for_func_status(result=True):
logger.error(
f"{operator} Installation status is not Succeeded after {timeout_install} seconds"
)
raise TimeoutExpiredError
def test_fio_with_block_storage(self):
name = 'test_workload'
spec = self.pod_obj.data.get('spec')
path = (
spec.get('containers')[0].get('volumeMounts')[0].get('mountPath')
)
work_load = 'fio'
storage_type = 'fs'
# few io parameters for Fio
runtime = 10
size = '200M'
wl = workload.WorkLoad(
name, path, work_load, storage_type, self.pod_obj
)
assert wl.setup()
io_params = templating.load_yaml_to_dict(constants.FIO_IO_PARAMS_YAML)
io_params['runtime'] = runtime
io_params['size'] = size
future_result = wl.run(**io_params)
timeout = 1200
sample = TimeoutSampler(
timeout=timeout, sleep=3, func=future_result.done
)
assert sample.wait_for_func_status(result=True)
try:
logger.info(future_result.result())
except exceptions.CommandFailed:
logger.exception(f"FIO failed")
raise
except Exception:
logger.exception(f"Found Exception")
raise
def reboot_rhv_vms(self, vms, timeout=600, wait=True, force=True):
"""
Reboot the RHV virtual machines
Args:
vms (list): Names of RHV vms
timeout (int): time in seconds to wait for VM to reboot
wait (bool): Wait for RHV VMs to reboot
force (bool): True to reboot vm forcibly False otherwise
"""
for vm in vms:
# Find the virtual machine
vm_service = self.get_vm_service(vm.id)
vm_service.reboot(force=force)
if wait:
# Wait till the virtual machine will start rebooting and then UP
try:
expc_status = types.VmStatus.REBOOT_IN_PROGRESS
look_up = False
for status in TimeoutSampler(timeout, 5,
self.get_vm_status, vm):
if not look_up and status != expc_status:
logger.info(
f"Waiting for RHV Machine {vm.name} to {expc_status}"
f" Current status is : {status}")
continue
elif not look_up and status == expc_status:
expc_status = types.VmStatus.UP
look_up = True
logger.info(
f"RHV Machine {vm} is now {expc_status} status"
)
continue
else:
logger.info(
f"Waiting for RHV Machine {vm.name} to Power ON"
f" Current status is : {status}")
if status == types.VmStatus.UP:
logger.info(
f"RHV Machine {vm.name} reached UP status")
break
except TimeoutExpiredError:
logger.error(
f"RHV VM {vm.name} is still not {expc_status} after "
f"initiating reboot")
raise
