def test_usage_of_default_storage_class(self):
"""Validate PVs creation for SC with default custom volname prefix"""
if get_openshift_version() < "3.9":
self.skipTest(
"'volumenameprefix' option for Heketi is not supported"
" in OCP older than 3.9")
# Unset 'default' option from all the existing Storage Classes
unset_sc_annotation_cmd = (
r"""oc annotate sc %s """
r""""storageclass%s.kubernetes.io/is-default-class"-""")
set_sc_annotation_cmd = (
r"""oc patch storageclass %s -p'{"metadata": {"annotations": """
r"""{"storageclass%s.kubernetes.io/is-default-class": "%s"}}}'""")
get_sc_cmd = (
r'oc get sc --no-headers '
r'-o=custom-columns=:.metadata.name,'
r':".metadata.annotations.storageclass\.'
r'kubernetes\.io\/is-default-class",:".metadata.annotations.'
r'storageclass\.beta\.kubernetes\.io\/is-default-class"')
sc_list = self.cmd_run(get_sc_cmd)
for sc in sc_list.split("\n"):
sc = sc.split()
if len(sc) != 3:
self.skipTest(
"Unexpected output for list of storage classes. "
"Following is expected to contain 3 keys:: %s" % sc)
for value, api_type in ((sc[1], ''), (sc[2], '.beta')):
if value == '<none>':
continue
self.cmd_run(unset_sc_annotation_cmd % (sc[0], api_type))
self.addCleanup(
self.cmd_run,
set_sc_annotation_cmd % (sc[0], api_type, value))
# Create new SC
prefix = "autotests-default-sc"
self.create_storage_class(sc_name_prefix=prefix)
# Make new SC be the default one and sleep for 1 sec to avoid races
self.cmd_run(set_sc_annotation_cmd % (self.sc_name, '', 'true'))
self.cmd_run(set_sc_annotation_cmd % (self.sc_name, '.beta', 'true'))
time.sleep(1)
# Create PVC without specification of SC
pvc_name = oc_create_pvc(
self.node, sc_name=None, pvc_name_prefix=prefix)
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(oc_delete, self.node, 'pvc', pvc_name)
# Wait for successful creation of PVC and check its SC
verify_pvc_status_is_bound(self.node, pvc_name)
get_sc_of_pvc_cmd = (
"oc get pvc %s --no-headers "
"-o=custom-columns=:.spec.storageClassName" % pvc_name)
out = self.cmd_run(get_sc_of_pvc_cmd)
self.assertEqual(out, self.sc_name)
def create_and_wait_for_pvcs(self, pvc_size=1,
pvc_name_prefix="autotests-pvc",
pvc_amount=1, sc_name=None,
timeout=120, wait_step=3):
node = self.ocp_client[0]
# Create storage class if not specified
if not sc_name:
if getattr(self, "sc_name", ""):
sc_name = self.sc_name
else:
sc_name = self.create_storage_class()
# Create PVCs
pvc_names = []
for i in range(pvc_amount):
pvc_name = oc_create_pvc(
node, sc_name, pvc_name_prefix=pvc_name_prefix,
pvc_size=pvc_size)
pvc_names.append(pvc_name)
self.addCleanup(
wait_for_resource_absence, node, 'pvc', pvc_name)
# Wait for PVCs to be in bound state
try:
for pvc_name in pvc_names:
verify_pvc_status_is_bound(node, pvc_name, timeout, wait_step)
finally:
if get_openshift_version() < "3.9":
reclaim_policy = "Delete"
else:
reclaim_policy = oc_get_custom_resource(
node, 'sc', ':.reclaimPolicy', sc_name)[0]
for pvc_name in pvc_names:
if reclaim_policy == 'Retain':
pv_name = get_pv_name_from_pvc(node, pvc_name)
self.addCleanup(oc_delete, node, 'pv', pv_name,
raise_on_absence=False)
custom = (r':.metadata.annotations."gluster\.kubernetes'
r'\.io\/heketi\-volume\-id"')
vol_id = oc_get_custom_resource(
node, 'pv', custom, pv_name)[0]
if self.sc.get('provisioner') == "kubernetes.io/glusterfs":
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url, vol_id,
raise_on_error=False)
else:
self.addCleanup(heketi_blockvolume_delete,
self.heketi_client_node,
self.heketi_server_url, vol_id,
raise_on_error=False)
self.addCleanup(oc_delete, node, 'pvc', pvc_name,
raise_on_absence=False)
return pvc_names
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg(
self):
"""Verify that block volume creation fails when we create block
volume of size greater than the default size of BHV.
Verify that block volume creation succeed when we create BHV
of size greater than the default size of BHV.
"""
default_bhv_size = get_default_block_hosting_volume_size(
self.node, self.heketi_dc_name)
reserve_size = default_bhv_size * 0.02
reserve_size = int(math.ceil(reserve_size))
self.verify_free_space(default_bhv_size + reserve_size + 2)
with self.assertRaises(ExecutionError):
# create a block vol greater than default BHV size
bvol_info = heketi_blockvolume_create(
self.heketi_client_node, self.heketi_server_url,
(default_bhv_size + 1), json=True)
self.addCleanup(
heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, bvol_info['id'])
sc_name = self.create_storage_class()
# create a block pvc greater than default BHV size
pvc_name = oc_create_pvc(
self.node, sc_name, pvc_size=(default_bhv_size + 1))
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False)
wait_for_events(
self.node, pvc_name, obj_type='PersistentVolumeClaim',
event_type='Warning', event_reason='ProvisioningFailed')
# create block hosting volume greater than default BHV size
vol_info = heketi_volume_create(
self.heketi_client_node, self.heketi_server_url,
(default_bhv_size + reserve_size + 2), block=True,
json=True)
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
# Cleanup PVC before block hosting volume to avoid failures
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False)
verify_pvc_status_is_bound(self.node, pvc_name)
def setUp(self):
"""Deploys, Verifies and adds resources required for testcases
in cleanup method
"""
self.oc_node = self.ocp_master_node[0]
self.prefix = "autotest-%s" % utils.get_random_str()
_storage_class = self.storage_classes.get(
'storage_class2',
self.storage_classes.get('block_storage_class'))
self.provisioner = _storage_class["provisioner"]
self.restsecretnamespace = _storage_class["restsecretnamespace"]
self.restuser = _storage_class["restuser"]
self.resturl = _storage_class["resturl"]
# using pvc size count as 1 by default
self.pvcsize = 1
# using pvc count as 10 by default
self.pvccount = 10
# create gluster block storage class, PVC and user app pod
self.sc_name, self.pvc_name, self.dc_name, self.secret_name = (
self.deploy_resouces()
)
# verify storage class
oc_get_yaml(self.oc_node, "sc", self.sc_name)
# verify pod creation, it's state and get the pod name
self.pod_name = get_pod_name_from_dc(
self.oc_node, self.dc_name, timeout=180, wait_step=3
)
wait_for_pod_be_ready(
self.oc_node, self.pod_name, timeout=180, wait_step=3
)
verify_pvc_status_is_bound(self.oc_node, self.pvc_name)
# create pvc's to test
self.pvc_list = []
for pvc in range(self.pvccount):
test_pvc_name = oc_create_pvc(
self.oc_node, self.sc_name,
pvc_name_prefix=self.prefix, pvc_size=self.pvcsize
)
self.pvc_list.append(test_pvc_name)
self.addCleanup(
wait_for_resource_absence, self.oc_node, "pvc", test_pvc_name,
timeout=600, interval=10
)
for pvc_name in self.pvc_list:
self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
def deploy_resouces(self):
"""Deploys required resources storage class, pvc and user app
with continous I/O runnig
Returns:
sc_name (str): deployed storage class name
pvc_name (str): deployed persistent volume claim name
dc_name (str): deployed deployment config name
secretname (str): created secret file name
"""
secretname = oc_create_secret(
self.oc_node, namespace=self.restsecretnamespace,
data_key=self.heketi_cli_key, secret_type=self.provisioner)
self.addCleanup(oc_delete, self.oc_node, 'secret', secretname)
sc_name = oc_create_sc(
self.oc_node,
sc_name_prefix=self.prefix, provisioner=self.provisioner,
resturl=self.resturl, restuser=self.restuser,
restsecretnamespace=self.restsecretnamespace,
restsecretname=secretname, volumenameprefix=self.prefix
)
self.addCleanup(oc_delete, self.oc_node, "sc", sc_name)
pvc_name = oc_create_pvc(
self.oc_node, sc_name,
pvc_name_prefix=self.prefix, pvc_size=self.pvcsize
)
self.addCleanup(
wait_for_resource_absence, self.oc_node, "pvc", pvc_name,
timeout=120, interval=5
)
self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
dc_name = oc_create_app_dc_with_io(
self.oc_node, pvc_name, dc_name_prefix=self.prefix
)
self.addCleanup(oc_delete, self.oc_node, "dc", dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.oc_node, dc_name, 0)
return sc_name, pvc_name, dc_name, secretname
def test_dynamic_provisioning_glusterblock_heketipod_failure(self):
"""Validate PVC with glusterblock creation when heketi pod is down"""
datafile_path = '/mnt/fake_file_for_%s' % self.id()
# Create DC with attached PVC
sc_name = self.create_storage_class()
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix='autotest-block', sc_name=sc_name)
app_1_dc_name, app_1_pod_name = self.create_dc_with_pvc(app_1_pvc_name)
# Write test data
write_data_cmd = ("dd if=/dev/urandom of=%s bs=1K count=100" %
datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
# Create second PVC
app_2_pvc_name = oc_create_pvc(self.node,
pvc_name_prefix='autotest-block2',
sc_name=sc_name)
self.addCleanup(wait_for_resource_absence, self.node, 'pvc',
app_2_pvc_name)
self.addCleanup(oc_delete, self.node, 'pvc', app_2_pvc_name)
# Create second app POD
app_2_dc_name = oc_create_app_dc_with_io(self.node, app_2_pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', app_2_dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, app_2_dc_name,
0)
app_2_pod_name = get_pod_name_from_dc(self.node, app_2_dc_name)
# Bring Heketi pod back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=2)
wait_for_pod_be_ready(self.node,
new_heketi_pod_name,
wait_step=5,
timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(self.node,
app_2_pod_name,
timeout=150,
wait_step=3)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg(
self):
"""Verify that block volume creation fails when we create block
volume of size greater than the default size of BHV.
Verify that block volume creation succeed when we create BHV
of size greater than the default size of BHV.
"""
default_bhv_size = get_default_block_hosting_volume_size(
self.node, self.heketi_dc_name)
reserve_size = default_bhv_size * 0.02
reserve_size = int(math.ceil(reserve_size))
self.verify_free_space(default_bhv_size + reserve_size + 2)
with self.assertRaises(ExecutionError):
# create a block vol greater than default BHV size
bvol_info = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
(default_bhv_size + 1),
json=True)
self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, bvol_info['id'])
sc_name = self.create_storage_class()
# create a block pvc greater than default BHV size
pvc_name = oc_create_pvc(self.node,
sc_name,
pvc_size=(default_bhv_size + 1))
self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(oc_delete,
self.node,
'pvc',
pvc_name,
raise_on_absence=False)
wait_for_events(self.node,
pvc_name,
obj_type='PersistentVolumeClaim',
event_type='Warning',
event_reason='ProvisioningFailed')
# create block hosting volume greater than default BHV size
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
(default_bhv_size + reserve_size + 2),
block=True,
json=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
# Cleanup PVC before block hosting volume to avoid failures
self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(oc_delete,
self.node,
'pvc',
pvc_name,
raise_on_absence=False)
verify_pvc_status_is_bound(self.node, pvc_name)
def test_targetcli_failure_during_block_pvc_creation(self):
h_node, h_server = self.heketi_client_node, self.heketi_server_url
# Disable redundant nodes and leave just 3 nodes online
h_node_id_list = heketi_node_list(h_node, h_server)
self.assertGreater(len(h_node_id_list), 2)
for node_id in h_node_id_list[3:]:
heketi_node_disable(h_node, h_server, node_id)
self.addCleanup(heketi_node_enable, h_node, h_server, node_id)
# Gather info about the Gluster node we are going to use for killing
# targetcli processes.
chosen_g_node_id = h_node_id_list[0]
chosen_g_node_info = heketi_node_info(h_node,
h_server,
chosen_g_node_id,
json=True)
chosen_g_node_ip = chosen_g_node_info['hostnames']['storage'][0]
chosen_g_node_hostname = chosen_g_node_info['hostnames']['manage'][0]
chosen_g_node_ip_and_hostname = set(
(chosen_g_node_ip, chosen_g_node_hostname))
g_pods = oc_get_custom_resource(
self.node,
'pod', [
':.metadata.name', ':.status.hostIP', ':.status.podIP',
':.spec.nodeName'
],
selector='glusterfs-node=pod')
if g_pods and g_pods[0]:
for g_pod in g_pods:
if chosen_g_node_ip_and_hostname.intersection(set(g_pod[1:])):
host_to_run_cmds = self.node
g_pod_prefix, g_pod = 'oc exec %s -- ' % g_pod[0], g_pod[0]
break
else:
err_msg = (
'Failed to find Gluster pod filtering it by following IPs '
'and hostnames: %s\nFound following Gluster pods: %s') % (
chosen_g_node_ip_and_hostname, g_pods)
g.log.error(err_msg)
raise AssertionError(err_msg)
else:
host_to_run_cmds, g_pod_prefix, g_pod = chosen_g_node_ip, '', ''
# Schedule deletion of targetcli process
file_for_bkp, pvc_number = "~/.targetcli/prefs.bin", 10
self.cmd_run("%scp %s %s_backup" %
(g_pod_prefix, file_for_bkp, file_for_bkp),
hostname=host_to_run_cmds)
self.addCleanup(self.cmd_run,
"%srm -f %s_backup" % (g_pod_prefix, file_for_bkp),
hostname=host_to_run_cmds)
kill_targetcli_services_cmd = (
"while true; do "
" %spkill targetcli || echo 'failed to kill targetcli process'; "
"done" % g_pod_prefix)
loop_for_killing_targetcli_process = g.run_async(
host_to_run_cmds, kill_targetcli_services_cmd, "root")
try:
# Create bunch of PVCs
sc_name, pvc_names = self.create_storage_class(), []
for i in range(pvc_number):
pvc_names.append(oc_create_pvc(self.node, sc_name, pvc_size=1))
self.addCleanup(wait_for_resources_absence, self.node, 'pvc',
pvc_names)
self.addCleanup(oc_delete, self.node, 'pvc', ' '.join(pvc_names))
# Check that we get expected number of provisioning errors
timeout, wait_step, succeeded_pvcs, failed_pvcs = 120, 1, [], []
_waiter, err_msg = Waiter(timeout=timeout, interval=wait_step), ""
for pvc_name in pvc_names:
_waiter._attempt = 0
for w in _waiter:
events = get_events(self.node,
pvc_name,
obj_type="PersistentVolumeClaim")
for event in events:
if event['reason'] == 'ProvisioningSucceeded':
succeeded_pvcs.append(pvc_name)
break
elif event['reason'] == 'ProvisioningFailed':
failed_pvcs.append(pvc_name)
break
else:
continue
break
if w.expired:
err_msg = (
"Failed to get neither 'ProvisioningSucceeded' nor "
"'ProvisioningFailed' statuses for all the PVCs in "
"time. Timeout was %ss, interval was %ss." %
(timeout, wait_step))
g.log.error(err_msg)
raise AssertionError(err_msg)
self.assertGreater(len(failed_pvcs), len(succeeded_pvcs))
finally:
# Restore targetcli workability
loop_for_killing_targetcli_process._proc.terminate()
# Revert breakage back which can be caused by BZ-1769426
check_bkp_file_size_cmd = ("%sls -lah %s | awk '{print $5}'" %
(g_pod_prefix, file_for_bkp))
bkp_file_size = self.cmd_run(check_bkp_file_size_cmd,
hostname=host_to_run_cmds).strip()
if bkp_file_size == "0":
self.cmd_run("%smv %s_backup %s" %
(g_pod_prefix, file_for_bkp, file_for_bkp),
hostname=host_to_run_cmds)
breakage_err_msg = (
"File located at '%s' was corrupted (zero size) on the "
"%s. Looks like BZ-1769426 took effect. \n"
"Don't worry, it has been restored after test failure." %
(file_for_bkp, "'%s' Gluster pod" % g_pod
if g_pod else "'%s' Gluster node" % chosen_g_node_ip))
g.log.error(breakage_err_msg)
if err_msg:
breakage_err_msg = "%s\n%s" % (err_msg, breakage_err_msg)
raise AssertionError(breakage_err_msg)
# Wait for all the PVCs to be in bound state
wait_for_pvcs_be_bound(self.node, pvc_names, timeout=300, wait_step=5)
def create_sc_with_parameter(self, vol_type, success=False, parameter={}):
"""creates storage class, pvc and validates event
Args:
vol_type (str): storage type either gluster file or block
success (bool): if True check for successfull else failure
for pvc creation event
parameter (dict): dictionary with storage class parameters
"""
if vol_type == "glusterfile":
sc = self.storage_classes.get(
'storage_class1',
self.storage_classes.get('file_storage_class'))
# Create secret file for usage in storage class
self.secret_name = oc_create_secret(
self.ocp_master_node[0],
namespace=sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(
oc_delete, self.ocp_master_node[0], 'secret', self.secret_name)
sc_parameter = {
"secretnamespace": sc['secretnamespace'],
"secretname": self.secret_name,
"volumetype": "replicate:3"
}
elif vol_type == "glusterblock":
sc = self.storage_classes.get(
'storage_class2',
self.storage_classes.get('block_storage_class'))
# Create secret file for usage in storage class
self.secret_name = oc_create_secret(
self.ocp_master_node[0],
namespace=sc.get('restsecretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=sc.get('provisioner', 'gluster.org/glusterblock'))
self.addCleanup(
oc_delete, self.ocp_master_node[0], 'secret', self.secret_name)
sc_parameter = {
"provisioner": "gluster.org/glusterblock",
"restsecretnamespace": sc['restsecretnamespace'],
"restsecretname": self.secret_name,
"hacount": sc['hacount']
}
else:
err_msg = "invalid vol_type %s" % vol_type
g.log.error(err_msg)
raise AssertionError(err_msg)
sc_parameter['resturl'] = sc['resturl']
sc_parameter['restuser'] = sc['restuser']
sc_parameter.update(parameter)
# Create storage class
self.sc_name = oc_create_sc(
self.ocp_master_node[0], **sc_parameter)
self.addCleanup(oc_delete, self.ocp_master_node[0], 'sc', self.sc_name)
# Create PVC
self.pvc_name = oc_create_pvc(self.ocp_client[0], self.sc_name)
self.addCleanup(
wait_for_resource_absence, self.ocp_master_node[0],
'pvc', self.pvc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0],
'pvc', self.pvc_name)
# Wait for event with error
event_reason = 'ProvisioningFailed'
if success:
event_reason = 'ProvisioningSucceeded'
wait_for_events(self.ocp_master_node[0],
obj_name=self.pvc_name,
obj_type='PersistentVolumeClaim',
event_reason=event_reason)
def test_create_delete_pvcs_to_make_gluster_reuse_released_space(self):
"""Validate reuse of volume space after deletion of PVCs"""
min_storage_gb = 10
# Set arbiter:disabled tags to the first 2 nodes
data_nodes = []
biggest_disks = []
self.assertGreater(len(self.node_id_list), 2)
for node_id in self.node_id_list[0:2]:
node_info = heketi_ops.heketi_node_info(
self.heketi_client_node, self.heketi_server_url,
node_id, json=True)
biggest_disk_free_space = 0
for device in node_info['devices']:
disk_free_space = int(device['storage']['free'])
if disk_free_space < (min_storage_gb * 1024**2):
self.skipTest(
"Devices are expected to have more than "
"%sGb of free space" % min_storage_gb)
if disk_free_space > biggest_disk_free_space:
biggest_disk_free_space = disk_free_space
self._set_arbiter_tag_with_further_revert(
self.heketi_client_node, self.heketi_server_url, 'device',
device['id'], 'disabled',
revert_to=device.get('tags', {}).get('arbiter'))
biggest_disks.append(biggest_disk_free_space)
self._set_arbiter_tag_with_further_revert(
self.heketi_client_node, self.heketi_server_url, 'node',
node_id, 'disabled',
revert_to=node_info.get('tags', {}).get('arbiter'))
data_nodes.append(node_info)
# Set arbiter:required tag to all other nodes and their devices
arbiter_nodes = []
for node_id in self.node_id_list[2:]:
node_info = heketi_ops.heketi_node_info(
self.heketi_client_node, self.heketi_server_url,
node_id, json=True)
for device in node_info['devices']:
self._set_arbiter_tag_with_further_revert(
self.heketi_client_node, self.heketi_server_url, 'device',
device['id'], 'required',
revert_to=device.get('tags', {}).get('arbiter'))
self._set_arbiter_tag_with_further_revert(
self.heketi_client_node, self.heketi_server_url, 'node',
node_id, 'required',
revert_to=node_info.get('tags', {}).get('arbiter'))
arbiter_nodes.append(node_info)
# Calculate size and amount of volumes to be created
pvc_size = int(min(biggest_disks) / 1024**2)
pvc_amount = max([len(n['devices']) for n in data_nodes]) + 1
# Create sc with gluster arbiter info
self.create_storage_class(is_arbiter_vol=True)
# Create and delete 3 small volumes concurrently
pvc_names = []
for i in range(3):
pvc_name = oc_create_pvc(
self.node, self.sc_name, pvc_name_prefix='arbiter-pvc',
pvc_size=int(pvc_size / 3))
pvc_names.append(pvc_name)
exception_exists = False
for pvc_name in pvc_names:
try:
verify_pvc_status_is_bound(self.node, pvc_name)
except Exception:
for pvc_name in pvc_names:
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
for pvc_name in pvc_names:
self.addCleanup(oc_delete, self.node, 'pvc', pvc_name)
exception_exists = True
if exception_exists:
raise
for pvc_name in pvc_names:
oc_delete(self.node, 'pvc', pvc_name)
for pvc_name in pvc_names:
wait_for_resource_absence(self.node, 'pvc', pvc_name)
# Create and delete big volumes in a loop
for i in range(pvc_amount):
pvc_name = oc_create_pvc(
self.node, self.sc_name, pvc_name_prefix='arbiter-pvc',
pvc_size=pvc_size)
try:
verify_pvc_status_is_bound(self.node, pvc_name)
except Exception:
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(oc_delete, self.node, 'pvc', pvc_name)
raise
oc_delete(self.node, 'pvc', pvc_name)
wait_for_resource_absence(self.node, 'pvc', pvc_name)
def test_dynamic_provisioning_glusterblock_heketipod_failure(self):
"""Validate PVC with glusterblock creation when heketi pod is down"""
datafile_path = '/mnt/fake_file_for_%s' % self.id()
# Create DC with attached PVC
sc_name = self.create_storage_class()
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix='autotest-block', sc_name=sc_name)
app_1_dc_name, app_1_pod_name = self.create_dc_with_pvc(app_1_pvc_name)
# Write test data
write_data_cmd = (
"dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
# Create second PVC
app_2_pvc_name = oc_create_pvc(
self.node, pvc_name_prefix='autotest-block2', sc_name=sc_name
)
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', app_2_pvc_name
)
# Create second app POD
app_2_dc_name = oc_create_app_dc_with_io(self.node, app_2_pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', app_2_dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.node, app_2_dc_name, 0)
app_2_pod_name = get_pod_name_from_dc(self.node, app_2_dc_name)
# Bring Heketi pod back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=2)
wait_for_pod_be_ready(
self.node, new_heketi_pod_name, wait_step=5, timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(
self.node, app_2_pod_name, timeout=150, wait_step=3)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_dev_path_block_volume_delete(self):
"""Validate device path name changes the deletion of
already existing file volumes
"""
pvc_size, pvc_amount = 2, 5
pvc_names, gluster_block_list, vol_details = [], [], []
# Fetch BHV list
h_bhv_list_before = heketi_ops.get_block_hosting_volume_list(
self.h_node, self.h_server).keys()
# Create storage class
sc_name = self.create_storage_class()
# Delete created BHV and BV as cleanup during failures
self.addCleanup(self._cleanup_heketi_volumes, h_bhv_list_before)
# Create PVC's
for i in range(0, pvc_amount):
pvc_name = openshift_ops.oc_create_pvc(self.node,
sc_name,
pvc_size=pvc_size)
pvc_names.append(pvc_name)
self.addCleanup(openshift_ops.wait_for_resource_absence, self.node,
'pvc', pvc_name)
self.addCleanup(openshift_ops.oc_delete,
self.node,
'pvc',
pvc_name,
raise_on_absence=False)
# Wait for PVC's to be bound
openshift_ops.wait_for_pvc_be_bound(self.node, pvc_names)
# Get volume name list
for pvc_name in pvc_names:
pv_name = openshift_ops.get_pv_name_from_pvc(self.node, pvc_name)
volume_name = openshift_ops.get_vol_names_from_pv(self.node,
pv_name,
vol_type='block')
vol_details.append(volume_name)
# Get BHV list after BV creation
h_bhv_list_after = heketi_ops.get_block_hosting_volume_list(
self.h_node, self.h_server).keys()
self.assertTrue(h_bhv_list_after, "Failed to get the BHV list")
# Validate BV's count
self.validate_block_volumes_count(self.h_node, self.h_server,
self.node_ip)
# Collect pvs info and detach disks and collect pvs info
pvs_info_before = openshift_storage_libs.get_pvs_info(
self.node, self.node_ip, self.devices_list, raise_on_error=False)
self.detach_and_attach_vmdk(self.vm_name, self.node_hostname,
self.devices_list)
pvs_info_after = openshift_storage_libs.get_pvs_info(
self.node, self.node_ip, self.devices_list, raise_on_error=False)
# Compare pvs info before and after
for (path, uuid, vg_name), (_path, _uuid,
_vg_name) in zip(pvs_info_before[:-1],
pvs_info_after[1:]):
self.assertEqual(
uuid, _uuid, "pv_uuid check failed. Expected:{},"
"Actual: {}".format(uuid, _uuid))
self.assertEqual(
vg_name, _vg_name, "vg_name check failed. Expected:"
"{}, Actual:{}".format(vg_name, _vg_name))
# Delete created PVC's
for pvc_name in pvc_names:
openshift_ops.oc_delete(self.node, 'pvc', pvc_name)
# Wait for pvc to get deleted
openshift_ops.wait_for_resources_absence(self.node, 'pvc', pvc_names)
# Get existing BHV list
for bhv_name in h_bhv_list_after:
b_list = block_libs.get_block_list(self.node_ip, volname=bhv_name)
self.assertIsNotNone(gluster_block_list,
"Failed to get gluster block list")
gluster_block_list.append(b_list)
# Get list of block volumes using heketi
h_blockvol_list = heketi_ops.heketi_blockvolume_list(self.h_node,
self.h_server,
json=True)
# Validate volumes created are not present
for vol in vol_details:
self.assertNotIn(vol, gluster_block_list,
"Failed to delete volume {}".format(vol))
self.assertNotIn(vol, h_blockvol_list['blockvolumes'],
"Failed to delete blockvolume '{}'".format(vol))
def test_dev_path_file_volume_delete(self):
"""Validate device path name changes the deletion of
already existing file volumes
"""
pvc_size, pvc_amount = 2, 5
vol_details, pvc_names = [], []
# Create PVC's
sc_name = self.create_storage_class()
for i in range(0, pvc_amount):
pvc_name = openshift_ops.oc_create_pvc(self.node,
sc_name,
pvc_size=pvc_size)
pvc_names.append(pvc_name)
self.addCleanup(openshift_ops.wait_for_resource_absence, self.node,
'pvc', pvc_name)
self.addCleanup(openshift_ops.oc_delete,
self.node,
'pvc',
pvc_name,
raise_on_absence=False)
# Wait for PVC's to be bound
openshift_ops.wait_for_pvcs_be_bound(self.node, pvc_names)
# Get Volumes name and validate volumes count
for pvc_name in pvc_names:
pv_name = openshift_ops.get_pv_name_from_pvc(self.node, pvc_name)
volume_name = openshift_ops.get_vol_names_from_pv(
self.node, pv_name)
vol_details.append(volume_name)
# Verify file volumes count
self.validate_file_volumes_count(self.h_node, self.h_server,
self.node_ip)
# Collect pvs info and detach disks and get pvs info
pvs_info_before = openshift_storage_libs.get_pvs_info(
self.node, self.node_ip, self.devices_list, raise_on_error=False)
self.detach_and_attach_vmdk(self.vm_name, self.node_hostname,
self.devices_list)
pvs_info_after = openshift_storage_libs.get_pvs_info(
self.node, self.node_ip, self.devices_list, raise_on_error=False)
# Compare pvs info before and after
for (path, uuid, vg_name), (_path, _uuid,
_vg_name) in zip(pvs_info_before[:-1],
pvs_info_after[1:]):
self.assertEqual(
uuid, _uuid, "pv_uuid check failed. Expected:{},"
"Actual: {}".format(uuid, _uuid))
self.assertEqual(
vg_name, _vg_name, "vg_name check failed. Expected:"
"{}, Actual:{}".format(vg_name, _vg_name))
# Delete created PVC's
for pvc_name in pvc_names:
openshift_ops.oc_delete(self.node, 'pvc', pvc_name)
# Wait for resource absence and get volume list
openshift_ops.wait_for_resources_absence(self.node, 'pvc', pvc_names)
vol_list = volume_ops.get_volume_list(self.node_ip)
self.assertIsNotNone(vol_list, "Failed to get volumes list")
# Validate volumes created are not present
for vol in vol_details:
self.assertNotIn(vol, vol_list,
"Failed to delete volume {}".format(vol))
def create_sc_with_parameter(self, vol_type, success=False, parameter={}):
"""creates storage class, pvc and validates event
Args:
vol_type (str): storage type either gluster file or block
success (bool): if True check for successfull else failure
for pvc creation event
parameter (dict): dictionary with storage class parameters
"""
if vol_type == "glusterfile":
sc = self.storage_classes.get(
'storage_class1',
self.storage_classes.get('file_storage_class'))
# Create secret file for usage in storage class
self.secret_name = oc_create_secret(
self.ocp_master_node[0],
namespace=sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(
oc_delete, self.ocp_master_node[0], 'secret', self.secret_name)
sc_parameter = {
"secretnamespace": sc['secretnamespace'],
"secretname": self.secret_name,
"volumetype": "replicate:3"
}
elif vol_type == "glusterblock":
sc = self.storage_classes.get(
'storage_class2',
self.storage_classes.get('block_storage_class'))
# Create secret file for usage in storage class
self.secret_name = oc_create_secret(
self.ocp_master_node[0],
namespace=sc.get('restsecretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=sc.get('provisioner', 'gluster.org/glusterblock'))
self.addCleanup(
oc_delete, self.ocp_master_node[0], 'secret', self.secret_name)
sc_parameter = {
"provisioner": "gluster.org/glusterblock",
"restsecretnamespace": sc['restsecretnamespace'],
"restsecretname": self.secret_name,
"hacount": sc['hacount']
}
else:
err_msg = "invalid vol_type %s" % vol_type
g.log.error(err_msg)
raise AssertionError(err_msg)
sc_parameter['resturl'] = sc['resturl']
sc_parameter['restuser'] = sc['restuser']
sc_parameter.update(parameter)
# Create storage class
self.sc_name = oc_create_sc(
self.ocp_master_node[0], **sc_parameter)
self.addCleanup(oc_delete, self.ocp_master_node[0], 'sc', self.sc_name)
# Create PVC
self.pvc_name = oc_create_pvc(self.ocp_client[0], self.sc_name)
self.addCleanup(
wait_for_resource_absence, self.ocp_master_node[0],
'pvc', self.pvc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0],
'pvc', self.pvc_name)
# Wait for event with error
event_reason = 'ProvisioningFailed'
if success:
event_reason = 'ProvisioningSucceeded'
wait_for_events(self.ocp_master_node[0],
obj_name=self.pvc_name,
obj_type='PersistentVolumeClaim',
event_reason=event_reason)
def create_and_wait_for_pvcs(
self, pvc_size=1, pvc_name_prefix="autotests-pvc", pvc_amount=1,
sc_name=None, timeout=600, wait_step=10, skip_waiting=False,
skip_cleanup=False):
"""Create multiple PVC's not waiting for it
Args:
pvc_size (int): size of PVC, default value is 1
pvc_name_prefix (str): volume prefix for each PVC, default value is
'autotests-pvc'
pvc_amount (int): number of PVC's, default value is 1
sc_name (str): storage class to create PVC, default value is None,
which will cause automatic creation of sc.
timeout (int): timeout time for waiting for PVC's to get bound
wait_step (int): waiting time between each try of PVC status check
skip_waiting (bool): boolean value which defines whether
we need to wait for PVC creation or not.
Returns:
List: list of PVC names
"""
node = self.ocp_client[0]
# Create storage class if not specified
if not sc_name:
if getattr(self, "sc_name", ""):
sc_name = self.sc_name
else:
sc_name = self.create_storage_class(skip_cleanup=skip_cleanup)
# Create PVCs
pvc_names = []
for i in range(pvc_amount):
pvc_name = oc_create_pvc(
node, sc_name, pvc_name_prefix=pvc_name_prefix,
pvc_size=pvc_size)
pvc_names.append(pvc_name)
if not skip_cleanup:
self.addCleanup(
wait_for_resources_absence, node, 'pvc', pvc_names)
# Wait for PVCs to be in bound state
try:
if not skip_waiting:
wait_for_pvcs_be_bound(node, pvc_names, timeout, wait_step)
finally:
if skip_cleanup:
return pvc_names
if get_openshift_version() < "3.9":
reclaim_policy = "Delete"
else:
reclaim_policy = oc_get_custom_resource(
node, 'sc', ':.reclaimPolicy', sc_name)[0]
for pvc_name in pvc_names:
if reclaim_policy == 'Retain':
pv_name = get_pv_name_from_pvc(node, pvc_name)
if not pv_name and skip_waiting:
continue
self.addCleanup(oc_delete, node, 'pv', pv_name,
raise_on_absence=False)
custom = (r':.metadata.annotations."gluster\.kubernetes'
r'\.io\/heketi\-volume\-id"')
vol_id = oc_get_custom_resource(
node, 'pv', custom, pv_name)[0]
if self.sc.get('provisioner') == "kubernetes.io/glusterfs":
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url, vol_id,
raise_on_error=False)
else:
self.addCleanup(heketi_blockvolume_delete,
self.heketi_client_node,
self.heketi_server_url, vol_id,
raise_on_error=False)
self.addCleanup(oc_delete, node, 'pvc', pvc_name,
raise_on_absence=False)
return pvc_names
def test_dynamic_provisioning_glusterfile_heketipod_failure(self):
"""Validate dynamic provisioning for gluster file when heketi pod down
"""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
sc_name = self.create_storage_class()
# Create PVC
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix="autotest-file", sc_name=sc_name)
# Create app POD with attached volume
app_1_pod_name = oc_create_tiny_pod_with_volume(
self.node,
app_1_pvc_name,
"test-pvc-mount-on-app-pod",
mount_path=mount_path,
image=self.io_container_image_cirros)
self.addCleanup(wait_for_resource_absence, self.node, 'pod',
app_1_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(self.node,
app_1_pod_name,
timeout=60,
wait_step=2)
# Write data to the app POD
write_data_cmd = ("dd if=/dev/urandom of=%s bs=1K count=100" %
datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
app_2_pvc_name = oc_create_pvc(self.node,
pvc_name_prefix="autotest-file2",
sc_name=sc_name)
self.addCleanup(wait_for_resource_absence, self.node, 'pvc',
app_2_pvc_name)
self.addCleanup(oc_delete,
self.node,
'pvc',
app_2_pvc_name,
raise_on_absence=False)
# Create second app POD
app_2_pod_name = oc_create_tiny_pod_with_volume(
self.node,
app_2_pvc_name,
"test-pvc-mount-on-app-pod",
mount_path=mount_path,
image=self.io_container_image_cirros)
self.addCleanup(wait_for_resource_absence, self.node, 'pod',
app_2_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name)
# Bring Heketi POD back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=2)
wait_for_pod_be_ready(self.node,
new_heketi_pod_name,
wait_step=5,
timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(self.node,
app_2_pod_name,
timeout=60,
wait_step=2)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_dynamic_provisioning_glusterfile_heketipod_failure(self):
"""Validate dynamic provisioning for gluster file when heketi pod down
"""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
sc_name = self.create_storage_class()
# Create PVC
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix="autotest-file", sc_name=sc_name
)
# Create app POD with attached volume
app_1_pod_name = oc_create_tiny_pod_with_volume(
self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', app_1_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(
self.node, app_1_pod_name, timeout=60, wait_step=2)
# Write data to the app POD
write_data_cmd = (
"dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
app_2_pvc_name = oc_create_pvc(
self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name
)
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False
)
# Create second app POD
app_2_pod_name = oc_create_tiny_pod_with_volume(
self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', app_2_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name)
# Bring Heketi POD back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=2)
wait_for_pod_be_ready(
self.node, new_heketi_pod_name, wait_step=5, timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(
self.node, app_2_pod_name, timeout=60, wait_step=2)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_create_delete_pvcs_to_make_gluster_reuse_released_space(self):
"""Validate reuse of volume space after deletion of PVCs"""
min_storage_gb = 10
# Set arbiter:disabled tags to the first 2 nodes
data_nodes = []
biggest_disks = []
self.assertGreater(len(self.node_id_list), 2)
for node_id in self.node_id_list[0:2]:
node_info = heketi_ops.heketi_node_info(self.heketi_client_node,
self.heketi_server_url,
node_id,
json=True)
biggest_disk_free_space = 0
for device in node_info['devices']:
disk_free_space = int(device['storage']['free'])
if disk_free_space < (min_storage_gb * 1024**2):
self.skipTest("Devices are expected to have more than "
"%sGb of free space" % min_storage_gb)
if disk_free_space > biggest_disk_free_space:
biggest_disk_free_space = disk_free_space
self._set_arbiter_tag_with_further_revert(
self.heketi_client_node,
self.heketi_server_url,
'device',
device['id'],
'disabled',
revert_to=device.get('tags', {}).get('arbiter'))
biggest_disks.append(biggest_disk_free_space)
self._set_arbiter_tag_with_further_revert(self.heketi_client_node,
self.heketi_server_url,
'node',
node_id,
'disabled',
revert_to=node_info.get(
'tags',
{}).get('arbiter'))
data_nodes.append(node_info)
# Set arbiter:required tag to all other nodes and their devices
arbiter_nodes = []
for node_id in self.node_id_list[2:]:
node_info = heketi_ops.heketi_node_info(self.heketi_client_node,
self.heketi_server_url,
node_id,
json=True)
for device in node_info['devices']:
self._set_arbiter_tag_with_further_revert(
self.heketi_client_node,
self.heketi_server_url,
'device',
device['id'],
'required',
revert_to=device.get('tags', {}).get('arbiter'))
self._set_arbiter_tag_with_further_revert(self.heketi_client_node,
self.heketi_server_url,
'node',
node_id,
'required',
revert_to=node_info.get(
'tags',
{}).get('arbiter'))
arbiter_nodes.append(node_info)
# Calculate size and amount of volumes to be created
pvc_size = int(min(biggest_disks) / 1024**2)
pvc_amount = max([len(n['devices']) for n in data_nodes]) + 1
# Create sc with gluster arbiter info
self.create_storage_class(is_arbiter_vol=True)
# Create and delete 3 small volumes concurrently
pvc_names = self.create_and_wait_for_pvcs(
pvc_size=int(pvc_size / 3),
pvc_name_prefix='arbiter-pvc',
pvc_amount=3,
sc_name=self.sc_name)
for pvc_name in pvc_names:
openshift_ops.oc_delete(self.node, 'pvc', pvc_name)
for pvc_name in pvc_names:
openshift_ops.wait_for_resource_absence(self.node, 'pvc', pvc_name)
# Create and delete big volumes in a loop
for i in range(pvc_amount):
pvc_name = openshift_ops.oc_create_pvc(
self.node,
self.sc_name,
pvc_name_prefix='arbiter-pvc',
pvc_size=pvc_size)
try:
openshift_ops.verify_pvc_status_is_bound(
self.node, pvc_name, 300, 10)
except Exception:
self.addCleanup(openshift_ops.wait_for_resource_absence,
self.node, 'pvc', pvc_name)
self.addCleanup(openshift_ops.oc_delete, self.node, 'pvc',
pvc_name)
raise
openshift_ops.oc_delete(self.node, 'pvc', pvc_name)
openshift_ops.wait_for_resource_absence(self.node, 'pvc', pvc_name)
def create_and_wait_for_pvcs(self,
pvc_size=1,
pvc_name_prefix="autotests-pvc",
pvc_amount=1,
sc_name=None,
timeout=120,
wait_step=3):
node = self.ocp_client[0]
# Create storage class if not specified
if not sc_name:
if getattr(self, "sc_name", ""):
sc_name = self.sc_name
else:
sc_name = self.create_storage_class()
# Create PVCs
pvc_names = []
for i in range(pvc_amount):
pvc_name = oc_create_pvc(node,
sc_name,
pvc_name_prefix=pvc_name_prefix,
pvc_size=pvc_size)
pvc_names.append(pvc_name)
self.addCleanup(wait_for_resource_absence, node, 'pvc', pvc_name)
# Wait for PVCs to be in bound state
try:
for pvc_name in pvc_names:
verify_pvc_status_is_bound(node, pvc_name, timeout, wait_step)
finally:
if get_openshift_version() < "3.9":
reclaim_policy = "Delete"
else:
reclaim_policy = oc_get_custom_resource(
node, 'sc', ':.reclaimPolicy', sc_name)[0]
for pvc_name in pvc_names:
if reclaim_policy == 'Retain':
pv_name = get_pv_name_from_pvc(node, pvc_name)
self.addCleanup(oc_delete,
node,
'pv',
pv_name,
raise_on_absence=False)
custom = (r':.metadata.annotations."gluster\.kubernetes'
r'\.io\/heketi\-volume\-id"')
vol_id = oc_get_custom_resource(node, 'pv', custom,
pv_name)[0]
if self.sc.get('provisioner') == "kubernetes.io/glusterfs":
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol_id,
raise_on_error=False)
else:
self.addCleanup(heketi_blockvolume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol_id,
raise_on_error=False)
self.addCleanup(oc_delete,
node,
'pvc',
pvc_name,
raise_on_absence=False)
return pvc_names
