def _validate_dal(values):
"""
Validates the move test. Checks for dal changes
:param values: dict with values to validate if they updated
:type values: dict
:return:
"""
# Fetch them from the dal
source_std = StoragedriverHelper.get_storagedriver_by_guid(values['source_std']['guid'])
target_std = StoragedriverHelper.get_storagedriver_by_guid(values['target_std']['guid']) # Should not have moved to here
for serialized_vdisk in values['vdisks']:
vdisk = VDiskHelper.get_vdisk_by_guid(serialized_vdisk['guid'])
# Expecting no changes in vdisks_guids
if vdisk.guid in source_std.vdisks_guids:
EdgeTester.LOGGER.info('Vdisks were not moved according the DAL.')
else:
raise ValueError('Vdisks were moved according the DAL.')
if vdisk.guid not in target_std.vdisks_guids:
EdgeTester.LOGGER.info('Vdisks were not moved to the target according the DAL.')
else:
raise ValueError('Vdisks guids were updated after move for target storagedriver.')
if vdisk.storagerouter_guid == source_std.storagerouter.guid:
EdgeTester.LOGGER.info('Owner has remained the same.')
else:
ValueError('Expected {0} but found {1} for vdisk.storagerouter_guid'.format(source_std.storagerouter.guid, vdisk.storagerouter_guid))
def _validate_move(values_to_check):
"""
Validates the move test. Checks IO, and checks for dal changes
:param values_to_check: dict with values to validate if they updated
:type values_to_check: dict
:return:
"""
# Fetch dal object
source_std = StoragedriverHelper.get_storagedriver_by_guid(
values_to_check['source_std']['guid'])
target_std = StoragedriverHelper.get_storagedriver_by_guid(
values_to_check['target_std']['guid'])
try:
MigrateTester._validate_dal(values_to_check)
except ValueError as ex:
MigrateTester.LOGGER.warning(
'DAL did not automatically change after a move. Got {0}'.
format(ex))
source_std.invalidate_dynamics([])
target_std.invalidate_dynamics([])
# Properties should have been reloaded
values_to_check[
'source_std'] = StoragedriverHelper.get_storagedriver_by_guid(
values_to_check['source_std']['guid']).serialize()
values_to_check[
'target_std'] = StoragedriverHelper.get_storagedriver_by_guid(
values_to_check['target_std']['guid']).serialize()
MigrateTester._validate_dal(values_to_check)
def _validate_dal(values):
"""
Validates the move test. Checks for dal changes
:param values: dict with values to validate if they updated
:type values: dict
:return:
"""
# Fetch them from the dal
source_std = StoragedriverHelper.get_storagedriver_by_guid(
values['source_std']['guid'])
target_std = StoragedriverHelper.get_storagedriver_by_guid(
values['target_std']['guid'])
vdisk = VDiskHelper.get_vdisk_by_guid(values['vdisk']['guid'])
if values['source_std'] == source_std.serialize():
# DAL values did not update - expecting a change in vdisks_guids
raise ValueError(
'Expecting the target Storagedriver to change but nothing happened...'
)
else:
# Expecting changes in vdisks_guids
if vdisk.guid in source_std.vdisks_guids:
raise ValueError(
'Vdisks guids were not updated after move for source storagedriver.'
)
else:
MigrateTester.LOGGER.info(
'All properties are updated for source storagedriver.')
if values['target_std'] == target_std.serialize():
raise ValueError(
'Expecting changes in the target Storagedriver but nothing changed.'
)
else:
if vdisk.guid not in target_std.vdisks_guids:
raise ValueError(
'Vdisks guids were not updated after move for target storagedriver.'
)
else:
MigrateTester.LOGGER.info(
'All properties are updated for target storagedriver.')
if values["vdisk"] == vdisk.serialize():
raise ValueError(
'Expecting changes in the vdisk but nothing changed.')
else:
if vdisk.storagerouter_guid == target_std.storagerouter.guid:
MigrateTester.LOGGER.info(
'All properties are updated for vdisk.')
else:
ValueError(
'Expected {0} but found {1} for vdisk.storagerouter_guid'.
format(vdisk.storagerouter_guid, vdisk.storagerouter_guid))
MigrateTester.LOGGER.info(
'Move vdisk was successful according to the dal (which fetches volumedriver info).'
)
def _validate_move(values_to_check):
"""
Validates the move test. Checks IO, and checks for dal changes
:param values_to_check: dict with values to validate if they updated
:type values_to_check: dict
:return: None
:rtype: NoneType
"""
source_std = StoragedriverHelper.get_storagedriver_by_guid(
values_to_check['source_std']['guid'])
source_std.invalidate_dynamics([])
for vdisk in values_to_check['vdisks']:
AdvancedDTLTester.LOGGER.info(
'Source is documented as {0} and vdisk is now on {1}'.format(
source_std.storagerouter.guid, vdisk.storagerouter_guid))
checks = 0
while checks <= AdvancedDTLTester.MIGRATE_CHECKS:
if vdisk.storagerouter_guid != source_std.storagerouter.guid:
AdvancedDTLTester.LOGGER.info(
'Move vdisk was successful according to the dal, source was {0} and destination is now {1}'
.format(source_std.storagerouter.guid,
vdisk.storagerouter_guid))
return
else:
AdvancedDTLTester.LOGGER.info(
'Move vdisk was NOT YET successful according to the dal, source was {0} and destination is now {1}, sleeping for {2} seconds'
.format(source_std.storagerouter.guid,
vdisk.storagerouter_guid,
AdvancedDTLTester.MIGRATE_TIMEOUT))
checks += 1
time.sleep(AdvancedDTLTester.MIGRATE_TIMEOUT)
raise ValueError("Move vdisk has FAILED!")
def setup_env(cls, domain_based=False):
"""
Return a dict containing instances of storagedrivers and storagerouters
:param domain_based:
:return: dict
"""
vpool = None
if domain_based:
destination_str, source_str, compute_str = StoragerouterHelper().get_storagerouters_by_role()
destination_storagedriver = None
source_storagedriver = None
if len(source_str.regular_domains) == 0:
storagedrivers = StoragedriverHelper.get_storagedrivers()
else:
storagedrivers = DomainHelper.get_storagedrivers_in_same_domain(domain_guid=source_str.regular_domains[0])
for storagedriver in storagedrivers:
if len(storagedriver.vpool.storagedrivers) < 2:
continue
if storagedriver.vpool.name not in cls.get_vpool_names():
continue
if storagedriver.guid in destination_str.storagedrivers_guids:
if destination_storagedriver is None and (source_storagedriver is None or source_storagedriver.vpool_guid == storagedriver.vpool_guid):
destination_storagedriver = storagedriver
cls.LOGGER.info('Chosen destination storagedriver is: {0}'.format(destination_storagedriver.storage_ip))
elif storagedriver.guid in source_str.storagedrivers_guids:
# Select if the source driver isn't select and destination is also unknown or the storagedriver has matches with the same vpool
if source_storagedriver is None and (destination_storagedriver is None or destination_storagedriver.vpool_guid == storagedriver.vpool_guid):
source_storagedriver = storagedriver
cls.LOGGER.info('Chosen source storagedriver is: {0}'.format(source_storagedriver.storage_ip))
assert source_storagedriver is not None and destination_storagedriver is not None, 'We require at least two storagedrivers within the same domain.'
else:
vpool = SetupHelper.get_vpool_with_2_storagedrivers()
available_storagedrivers = [storagedriver for storagedriver in vpool.storagedrivers]
destination_storagedriver = available_storagedrivers.pop(random.randrange(len(available_storagedrivers)))
source_storagedriver = available_storagedrivers.pop(random.randrange(len(available_storagedrivers)))
destination_str = destination_storagedriver.storagerouter # Will act as volumedriver node
source_str = source_storagedriver.storagerouter # Will act as volumedriver node
compute_str = [storagerouter for storagerouter in StoragerouterHelper.get_storagerouters() if
storagerouter.guid not in [destination_str.guid, source_str.guid]][0] # Will act as compute node
# Choose source & destination storage driver
destination_storagedriver = [storagedriver for storagedriver in destination_str.storagedrivers if storagedriver.vpool_guid == vpool.guid][0]
source_storagedriver = [storagedriver for storagedriver in source_str.storagedrivers if storagedriver.vpool_guid == vpool.guid][0]
cls.LOGGER.info('Chosen destination storagedriver is: {0}'.format(destination_storagedriver.storage_ip))
cls.LOGGER.info('Chosen source storagedriver is: {0}'.format(source_storagedriver.storage_ip))
cluster_info = {'storagerouters': {'destination': destination_str,
'source': source_str,
'compute': compute_str},
'storagedrivers': {'destination': destination_storagedriver,
'source': source_storagedriver},
'vpool': vpool}
return cluster_info
def setup(cls, logger=LOGGER):
destination_str, source_str, compute_str = StoragerouterHelper().get_storagerouters_by_role()
destination_storagedriver = None
source_storagedriver = None
if len(source_str.regular_domains) == 0:
storagedrivers = StoragedriverHelper.get_storagedrivers()
else:
storagedrivers = DomainHelper.get_storagedrivers_in_same_domain(domain_guid=source_str.regular_domains[0])
for storagedriver in storagedrivers:
if len(storagedriver.vpool.storagedrivers) < 2:
continue
if storagedriver.guid in destination_str.storagedrivers_guids:
if destination_storagedriver is None and (source_storagedriver is None or source_storagedriver.vpool_guid == storagedriver.vpool_guid):
destination_storagedriver = storagedriver
logger.info('Chosen destination storagedriver is: {0}'.format(destination_storagedriver.storage_ip))
elif storagedriver.guid in source_str.storagedrivers_guids:
# Select if the source driver isn't select and destination is also unknown or the storagedriver has matches with the same vpool
if source_storagedriver is None and (destination_storagedriver is None or destination_storagedriver.vpool_guid == storagedriver.vpool_guid):
source_storagedriver = storagedriver
logger.info('Chosen source storagedriver is: {0}'.format(source_storagedriver.storage_ip))
assert source_storagedriver is not None and destination_storagedriver is not None, 'We require at least two storagedrivers within the same domain.'
cluster_info = {'storagerouters': {'destination': destination_str, 'source': source_str, 'compute': compute_str},
'storagedrivers': {'destination': destination_storagedriver, 'source': source_storagedriver}}
compute_client = SSHClient(compute_str, username='******')
is_ee = SystemHelper.get_ovs_version(source_str) == 'ee'
if is_ee is True:
fio_bin_loc = cls.FIO_BIN_EE['location']
fio_bin_url = cls.FIO_BIN_EE['url']
else:
fio_bin_loc = cls.FIO_BIN['location']
fio_bin_url = cls.FIO_BIN['url']
compute_client.run(['wget', fio_bin_url, '-O', fio_bin_loc])
compute_client.file_chmod(fio_bin_loc, 755)
return cluster_info, is_ee, fio_bin_loc
def _get_storagedriver_information(self, storage_ip=None):
"""
Returns a storagedriver its information
Currently a random storagedriver is chosen from the vpool if storage_ip is None,
in the future this can be strategically chosen
:return: storagedriver information
:rtype: dict
"""
api = self._setup_ovs_client()
data = StoragedriverHelper.get_storagedrivers_by_vpoolguid(
vpool_guid=self.vpool_guid, api=api)
# if a specific storagedriver is requested, provide this one. else provide a random one
if storage_ip:
storagedriver = [
sd for sd in data if sd['storage_ip'] == storage_ip
][0]
else:
storagedriver = random.choice(data)
LOG.debug(
'libovsvolumedriver.get_storagedriver_information {0}'.format(
storagedriver))
return storagedriver
def _execute_test(cls):
"""
Validate if DTL is configured as desired
REQUIREMENTS:
* 1 vPool should be available with 1 storagedriver
* 1 vPool should be available with 2 or more storagedrivers in 2 separate domains
OPTIONAL:
* 1 vPool with 1 storagedriver with disabled DTL
:return:
"""
cls.LOGGER.info("Starting to validate the basic DTL")
##########################
# get deployment details #
##########################
vpools = VPoolHelper.get_vpools()
assert len(vpools) >= 1, "Not enough vPools to test"
# Get a suitable vpools
vpool_single_sd = None
vpool_multi_sd = None
vpool_dtl_disabled = None
for vp in VPoolHelper.get_vpools():
if vp.configuration['dtl_mode'] != VPoolHelper.DtlStatus.DISABLED:
if len(vp.storagedrivers) == 1 and vpool_single_sd is None:
vpool_single_sd = vp
cls.LOGGER.info(
"vPool `{0}` has been chosen for SINGLE vPool DTL tests"
.format(vp.name))
elif len(vp.storagedrivers) >= 2 and vpool_multi_sd is None:
vpool_multi_sd = vp
cls.LOGGER.info(
"vPool `{0}` has been chosen for MULTI vPool DTL tests"
.format(vp.name))
else:
cls.LOGGER.info(
"vPool `{0}` is not suited for tests".format(vp.name))
else:
cls.LOGGER.info(
"vPool `{0}` with DISABLED DTL is available and will be tested!"
.format(vp.name))
vpool_dtl_disabled = vp
assert vpool_single_sd is not None, "A vPool should be available with 1 storagedriver"
assert vpool_multi_sd is not None, "A vPool should be available with 2 or more storagedrivers"
# pick a random storagedriver
storagedriver_single = vpool_single_sd.storagedrivers[0]
storagedriver_multi = random.choice(vpool_multi_sd.storagedrivers)
storagedrivers = [storagedriver_single, storagedriver_multi]
# check disabled DTL
storagedriver_disabled_dtl = None
if vpool_dtl_disabled is not None:
storagedriver_disabled_dtl = random.choice(
vpool_dtl_disabled.storagedrivers)
storagedrivers.append(storagedriver_disabled_dtl)
# key = amount of storagedrivers or a_s
# value = list with the vpool & storagedriver to test
vpools_to_test = {
1: [{
"vpool": vpool_single_sd,
"storagedriver": storagedriver_single
}],
2: [{
"vpool": vpool_multi_sd,
"storagedriver": storagedriver_multi
}]
}
# check if disabled DTL vpool needs to be added
if vpool_dtl_disabled is not None:
a_s = len(vpool_dtl_disabled.storagedrivers)
v_s = {
"vpool": vpool_dtl_disabled,
"storagedriver": storagedriver_disabled_dtl
}
if a_s in vpools_to_test:
vpools_to_test[a_s].append(v_s)
else:
vpools_to_test[a_s] = [v_s]
##############
# start test #
##############
for a_s, vpools in vpools_to_test.iteritems():
start = time.time()
for vpool in vpools:
cls.LOGGER.info(
"Starting DTL test with vPool {0} and {1} storagedrivers".
format(vpool['vpool'].name,
len(vpool['vpool'].storagedrivers)))
vdisk_name = "{0}-{1}-{2}".format(
cls.VDISK_NAME, vpool['vpool'].name,
str(len(vpool['vpool'].storagedrivers)))
try:
vdisk_guid = VDiskSetup.create_vdisk(
vdisk_name=vdisk_name + '.raw',
vpool_name=vpool['vpool'].name,
size=cls.SIZE_VDISK,
storagerouter_ip=vpool['storagedriver'].storagerouter.
ip)
# Fetch to validate if it was properly created
vdisk = VDiskHelper.get_vdisk_by_guid(
vdisk_guid=vdisk_guid)
except TimeOutError:
cls.LOGGER.error("Creation of the vDisk has timed out.")
raise
except RuntimeError as ex:
cls.LOGGER.info("Creation of vDisk failed: {0}".format(ex))
raise
else:
#####################################
# check DTL status after deployment #
#####################################
correct_msg = "vDisk {0} with {1} storagedriver(s) has correct DTL status: ".format(
vdisk_name, a_s)
if a_s == 1 and vdisk.dtl_status == VDiskHelper.DtlStatus.STANDALONE:
cls.LOGGER.info(correct_msg + vdisk.dtl_status)
elif a_s >= 2 and vdisk.dtl_status == VDiskHelper.DtlStatus.SYNC:
cls.LOGGER.info(correct_msg + vdisk.dtl_status)
elif vdisk.dtl_status == VDiskHelper.DtlStatus.DISABLED and vpool[
'vpool'].configuration[
'dtl_mode'] == VPoolHelper.DtlStatus.DISABLED:
cls.LOGGER.info(
correct_msg +
" Note: vdisk DTL is disabled but vPool DTL is also disabled!"
)
else:
error_msg = "vDisk {0} with {1} storagedriver(s) has WRONG DTL status: {2}".format(
vdisk_name, a_s, vdisk.dtl_status)
cls.LOGGER.error(error_msg)
raise RuntimeError(error_msg)
################################
# try to change the DTL config #
################################
base_config = {
"sco_size": 4,
"dtl_mode": VPoolHelper.DtlStatus.SYNC,
"write_buffer": 512
}
if a_s == 1:
########################################################################################
# change config to domain with non existing storagedrivers of this vpool (should fail) #
########################################################################################
cls.LOGGER.info(
"Starting test: change config to domain with non existing storagedrivers "
"of this vpool (should fail)")
base_config['dtl_target'] = [
random.choice([
domain_guid for domain_guid in
DomainHelper.get_domain_guids()
if domain_guid not in vpool['storagedriver'].
storagerouter.regular_domains
])
]
cls.LOGGER.info("Changing dtl_target to: {0}".format(
DomainHelper.get_domain_by_guid(
domain_guid=base_config['dtl_target']
[0]).name))
try:
cls.LOGGER.info(base_config)
VDiskSetup.set_config_params(
vdisk_name=vdisk_name + '.raw',
vpool_name=vpool['vpool'].name,
config=base_config)
error_msg = "Changing config to a domain with non existing storagedrivers should have failed with vdisk: {0}!".format(
vdisk_name)
cls.LOGGER.error(error_msg)
raise Exception(error_msg)
except TimeOutError:
cls.LOGGER.error(
"Changing config to a same domain with only 1 storagedriver has timed out."
)
raise
except RuntimeError:
cls.LOGGER.info(
"Changing config to a domain with non existing storagedrivers has failed as expected!"
)
##############################################################################################
# change config to domain where there are other storagedrivers but not of ours (should fail) #
##############################################################################################
cls.LOGGER.info(
"Starting test: change config to domain where there are other storagedrivers but not of ours (should fail)"
)
filtered_domains = list(
set(DomainHelper.get_domain_guids()) -
set(vpool['storagedriver'].storagerouter.
regular_domains))
base_config['dtl_target'] = [filtered_domains[0]]
cls.LOGGER.info(
"Current vdisk domain location: {0}".format(
DomainHelper.get_domain_by_guid(
domain_guid=vpool['storagedriver'].
storagerouter.regular_domains[0]).name))
cls.LOGGER.info("Changing dtl_target to: {0}".format(
DomainHelper.get_domain_by_guid(
domain_guid=base_config['dtl_target']
[0]).name))
try:
VDiskSetup.set_config_params(
vdisk_name=vdisk_name + '.raw',
vpool_name=vpool['vpool'].name,
config=base_config)
error_msg = "Changing config to a same domain with only 1 storagedriver should have failed with vdisk: {0}!".format(
vdisk_name)
cls.LOGGER.error(error_msg)
raise Exception(error_msg)
except TimeOutError:
cls.LOGGER.error(
"Changing config to a domain with non existing storagedrivers has timed out."
)
raise
except RuntimeError:
cls.LOGGER.info(
"Changing config to a same domain with only 1 storagedriver has failed as expected!"
)
elif a_s >= 2:
#######################################################################
# change config to domain with active storagedrivers (should succeed) #
#######################################################################
cls.LOGGER.info(
"Starting test: change config to domain with active storagedrivers (should succeed)"
)
# change current target domain to other target domain
current_vdisk_domains = StoragedriverHelper.get_storagedriver_by_id(
storagedriver_id=vdisk.storagedriver_id
).storagerouter.regular_domains
cls.LOGGER.info(
"Currently the vdisk is living in: {0}".format(
current_vdisk_domains))
vpool_domains = VPoolHelper.get_domains_by_vpool(
vpool_name=vdisk.vpool.name)
cls.LOGGER.info(
"Currently the vpool {0} is available in: {1}".
format(vdisk.vpool.name, vpool_domains))
future_domains = list(
set(vpool_domains) - set(current_vdisk_domains))
cls.LOGGER.info(
"DTL will be moved to other domain: {0}".format(
future_domains))
base_config['dtl_target'] = future_domains
# change settings
try:
VDiskSetup.set_config_params(
vdisk_name=vdisk_name + '.raw',
vpool_name=vpool['vpool'].name,
config=base_config)
except TimeOutError:
cls.LOGGER.error(
"Changing config to a same domain with only 1 storagedriver has timed out."
)
raise
except RuntimeError:
cls.LOGGER.error(
"Changing config to a same domain with only 1 storagedriver was unsuccesful!"
)
raise
cls.LOGGER.info(
"Changing config to a same domain with only 1 storagedriver was successful!"
)
cls.LOGGER.info("Removing vDisk {0}".format(vdisk.name))
VDiskRemover.remove_vdisk(vdisk_guid=vdisk.guid)
cls.LOGGER.info("Finished removing vDisk {0}".format(
vdisk.name))
end = time.time()
# display run time
cls.LOGGER.info("Run testing the DTL took {0} seconds".format(
int(end - start)))
cls.LOGGER.info("Finished to validate the basic DTL")