def setup_env(cls, domain_based=False):
"""
Return a dict containing instances of storagedrivers and storagerouters
:param domain_based:
:return: dict
"""
vpool = None
if domain_based:
destination_str, source_str, compute_str = StoragerouterHelper().get_storagerouters_by_role()
destination_storagedriver = None
source_storagedriver = None
if len(source_str.regular_domains) == 0:
storagedrivers = StoragedriverHelper.get_storagedrivers()
else:
storagedrivers = DomainHelper.get_storagedrivers_in_same_domain(domain_guid=source_str.regular_domains[0])
for storagedriver in storagedrivers:
if len(storagedriver.vpool.storagedrivers) < 2:
continue
if storagedriver.vpool.name not in cls.get_vpool_names():
continue
if storagedriver.guid in destination_str.storagedrivers_guids:
if destination_storagedriver is None and (source_storagedriver is None or source_storagedriver.vpool_guid == storagedriver.vpool_guid):
destination_storagedriver = storagedriver
cls.LOGGER.info('Chosen destination storagedriver is: {0}'.format(destination_storagedriver.storage_ip))
elif storagedriver.guid in source_str.storagedrivers_guids:
# Select if the source driver isn't select and destination is also unknown or the storagedriver has matches with the same vpool
if source_storagedriver is None and (destination_storagedriver is None or destination_storagedriver.vpool_guid == storagedriver.vpool_guid):
source_storagedriver = storagedriver
cls.LOGGER.info('Chosen source storagedriver is: {0}'.format(source_storagedriver.storage_ip))
assert source_storagedriver is not None and destination_storagedriver is not None, 'We require at least two storagedrivers within the same domain.'
else:
vpool = SetupHelper.get_vpool_with_2_storagedrivers()
available_storagedrivers = [storagedriver for storagedriver in vpool.storagedrivers]
destination_storagedriver = available_storagedrivers.pop(random.randrange(len(available_storagedrivers)))
source_storagedriver = available_storagedrivers.pop(random.randrange(len(available_storagedrivers)))
destination_str = destination_storagedriver.storagerouter # Will act as volumedriver node
source_str = source_storagedriver.storagerouter # Will act as volumedriver node
compute_str = [storagerouter for storagerouter in StoragerouterHelper.get_storagerouters() if
storagerouter.guid not in [destination_str.guid, source_str.guid]][0] # Will act as compute node
# Choose source & destination storage driver
destination_storagedriver = [storagedriver for storagedriver in destination_str.storagedrivers if storagedriver.vpool_guid == vpool.guid][0]
source_storagedriver = [storagedriver for storagedriver in source_str.storagedrivers if storagedriver.vpool_guid == vpool.guid][0]
cls.LOGGER.info('Chosen destination storagedriver is: {0}'.format(destination_storagedriver.storage_ip))
cls.LOGGER.info('Chosen source storagedriver is: {0}'.format(source_storagedriver.storage_ip))
cluster_info = {'storagerouters': {'destination': destination_str,
'source': source_str,
'compute': compute_str},
'storagedrivers': {'destination': destination_storagedriver,
'source': source_storagedriver},
'vpool': vpool}
return cluster_info
def create_volume_from_snapshot(self, volume, snapshot):
"""
Creates a volume from a snapshot.
:param volume: new volume object
:param snapshot: existing snapshot object
"""
new_volume_name = self.VOLUME_PREFIX + str(volume.id)
api = self._setup_ovs_client()
# pick a random storagerouter to deploy the new clone on
storagerouter_ip = random.choice([
sr
for sr in StoragerouterHelper.get_storagerouters(api=api).values()
if self.vpool_guid in sr['vpools_guids']
])
VDiskSetup.create_clone(new_vdisk_name=new_volume_name,
storagerouter_ip=storagerouter_ip,
api=api,
vpool_guid=self.vpool_guid,
snapshot_name=snapshot['name'])
# return volume location
storage_ip = VDiskHelper.get_vdisk_location_by_name(
vdisk_name=new_volume_name, vpool_guid=self.vpool_guid,
api=api)['storage_ip']
location = self._get_volume_location(volume_name=new_volume_name,
storage_ip=storage_ip)
volume['provider_location'] = location
LOG.debug('libovsvolumedriver.ovs_clone_from_snapshot {0} '.format(
volume.id))
return {'provider_location': volume['provider_location']}
def create_cloned_volume(self, volume, src_vref):
"""
Creates a clone of the specified volume.
:param volume: new volume object
:param src_vref: source volume object
"""
source_volume_name = self.VOLUME_PREFIX + str(src_vref.id)
new_volume_name = self.VOLUME_PREFIX + str(volume.id)
LOG.debug('libovsvolumedriver.ovs_clone new={0} source={1}'.format(
volume.id, src_vref.id))
api = self._setup_ovs_client()
# pick a random storagerouter to deploy the new clone on
storagerouter_ip = random.choice([
sr
for sr in StoragerouterHelper.get_storagerouters(api=api).values()
if self.vpool_guid in sr['vpools_guids']
])
VDiskSetup.create_clone(vdisk_name=source_volume_name,
new_vdisk_name=new_volume_name,
storagerouter_ip=storagerouter_ip,
api=api,
vpool_guid=self.vpool_guid)
# return volume location
storage_ip = VDiskHelper.get_vdisk_location_by_name(
vdisk_name=new_volume_name, vpool_guid=self.vpool_guid,
api=api)['storage_ip']
location = self._get_volume_location(volume_name=new_volume_name,
storage_ip=storage_ip)
volume['provider_location'] = location
return {'provider_location': volume['provider_location']}
def _get_test_name():
"""
Retrieve a structured environment test name
:returns: a structured environment based test name
:rtype: str
"""
number_of_nodes = len(StoragerouterHelper.get_storagerouters())
split_ip = System.get_my_storagerouter().ip.split('.')
return str(number_of_nodes) + 'N-' + split_ip[2] + '.' + split_ip[3]
def validate_basic_log_rotate():
"""
Validate that a basic logrotate script works
:return:
"""
LogrotateChecks.LOGGER.info('Starting validating basic logrotate')
storagerouter_ips = StoragerouterHelper.get_storagerouter_ips()
assert len(
storagerouter_ips) >= 1, "We need at least 1 storagerouters!"
def _set_mds_safety(vpool, safety=None, checkup=False, logger=LOGGER):
if safety is None:
safety = len(StoragerouterHelper.get_storagerouters())
if safety <= 0:
raise ValueError('Safety should be at least 1.')
logger.debug('Setting the safety to {} and {} checkup'.format(safety, 'will' if checkup is True else 'false'))
storagedriver_config = Configuration.get('/ovs/vpools/{0}/mds_config'.format(vpool.guid))
current_safety = storagedriver_config
current_safety['mds_safety'] = safety
Configuration.set('/ovs/framework/storagedriver', current_safety)
if checkup is True:
MDSServiceController.mds_checkup()
def validate_add_append_remove_roles():
"""
Validate a add role, remove roles and append
You need at least 1 free partition on a storagerouter
:return:
"""
RoleChecks.LOGGER.info('Starting validating add-append-remove roles')
storagerouter_ips = StoragerouterHelper.get_storagerouter_ips()
assert len(
storagerouter_ips) >= 1, "We need at least 1 storagerouters!"
def validate_ssh():
"""
Validate if ssh keys are distributed between nodes
:return:
"""
SshChecks.LOGGER.info('Starting validating SSH keys')
storagerouter_ips = StoragerouterHelper.get_storagerouter_ips()
assert len(
storagerouter_ips) >= 2, "We need at least 2 storagerouters!"
issues_found = []
for user in SshChecks.CHECK_USERS:
for env_ip_from in storagerouter_ips:
client = SSHClient(env_ip_from, username=user)
cwd = client.run(['pwd'])
# Check if the home dir is opt/OpenvStorage
if cwd == 'opt/OpenvStorage':
out = client.run(["cat", "./.ssh/known_hosts"])
for env_ip_to in storagerouter_ips:
if env_ip_from != env_ip_to:
if env_ip_to not in out:
error_msg = "Host key verification NOT FOUND between `{0}` and `{1}` for user `{2}`"\
.format(env_ip_from, env_ip_to, user)
SshChecks.LOGGER.error(error_msg)
issues_found.append(error_msg)
else:
SshChecks.LOGGER.info(
"Host key verification found between `{0}` and `{1}` for user `{2}`"
.format(env_ip_from, env_ip_to, user))
else:
SshChecks.LOGGER.error(
"Could not open ~/.ssh/known_hosts. Current working directory for user {0} is {1}"
.format(user, cwd))
if len(issues_found) != 0:
raise RuntimeError(
"One or more hosts keys not found on certain nodes! "
"Please check /var/log/ovs/scenario.log for more information!")
SshChecks.LOGGER.info('Finished validating SSH keys')
def __init__(self):
self.LOGGER.info("Initializing concurrent vpool testing.")
self.valid_storagerouters = []
for storagerouter in StoragerouterHelper.get_storagerouters():
try:
RoleValidation.check_required_roles(
VPoolSetup.REQUIRED_VPOOL_ROLES, storagerouter.ip, "LOCAL")
self.valid_storagerouters.append(storagerouter)
self.LOGGER.info(
'Added storagerouter {0} to list of eligible storagerouters'
.format(storagerouter.ip))
except RuntimeError:
pass
assert len(
self.valid_storagerouters
) >= 1, 'At least one storagerouter with valid roles required: none found!'
self.alba_backends = BackendHelper.get_alba_backends()
assert len(self.alba_backends
) >= 1, 'At least one backend required, none found!'
self.vpools = []
super(VPoolTester, self).__init__()
def restore_backup(self, context, backup, volume, backup_service):
"""
Restore an existing backup to a new or existing volume.
"""
volume_name = self.VOLUME_PREFIX + str(volume.id)
backup_snapshot_name = self.SNAPSHOT_BACKUP_PREFIX + str(backup['id'])
LOG.debug('libovsvolumedriver.ovs_backup_restore {0} {1}'.format(
volume_name, backup_snapshot_name))
api = self._setup_ovs_client()
snapshot, vdisk_guid, vdisk_name = VDiskHelper.get_snapshot_by_name(
snapshot_name=backup_snapshot_name,
vpool_guid=self.vpool_guid,
api=api)
if vdisk_name == volume_name:
# rollback
VDiskSetup.rollback_to_snapshot(vpool_guid=self.vpool_guid,
snapshot_name=backup_snapshot_name,
api=api)
else:
# to new volume
storagerouter_ip = random.choice([
sr for sr in StoragerouterHelper.get_storagerouters(
api=api).values() if self.vpool_guid in sr['vpools_guids']
])
VDiskSetup.create_clone(vpool_guid=self.vpool_guid,
new_vdisk_name=volume_name,
storagerouter_ip=storagerouter_ip,
api=api,
snapshot_name=backup_snapshot_name)
# return volume location
storage_ip = VDiskHelper.get_vdisk_location_by_name(
vdisk_name=volume_name, vpool_guid=self.vpool_guid,
api=api)['storage_ip']
location = self._get_volume_location(volume_name=volume_name,
storage_ip=storage_ip)
volume['provider_location'] = location
return {'provider_location': volume['provider_location']}
def validate_services(tries=SERVICE_TRIES, timeout=SERVICE_TIMEOUT):
"""
Validate if all services come up after installation of the setup
:param tries: amount of tries to check if ovs services are running
:type tries: int
:param timeout: timeout between tries
:type timeout: int
:return:
"""
ServiceChecks.LOGGER.info('Starting validating services')
storagerouter_ips = StoragerouterHelper.get_storagerouter_ips()
assert len(
storagerouter_ips) >= 1, "We need at least 1 storagerouters!"
# commence test
for storagerouter_ip in storagerouter_ips:
ServiceChecks.LOGGER.info(
'Starting service check on node `{0}`'.format(
storagerouter_ip))
amount_tries = 0
non_running_services = None
client = SSHClient(storagerouter_ip, username='******')
while tries >= amount_tries:
non_running_services = SystemHelper.get_non_running_ovs_services(
client)
if len(non_running_services) == 0:
break
else:
amount_tries += 1
time.sleep(timeout)
assert len(
non_running_services
) == 0, "Found non running services `{0}` after reboot on node `{1}`".format(
non_running_services, storagerouter_ip)
ServiceChecks.LOGGER.info(
'Finished validating services on node `{0}`'.format(
storagerouter_ip))
ServiceChecks.LOGGER.info('Finished validating services')
def setup(cls, logger=LOGGER):
destination_str, source_str, compute_str = StoragerouterHelper().get_storagerouters_by_role()
destination_storagedriver = None
source_storagedriver = None
if len(source_str.regular_domains) == 0:
storagedrivers = StoragedriverHelper.get_storagedrivers()
else:
storagedrivers = DomainHelper.get_storagedrivers_in_same_domain(domain_guid=source_str.regular_domains[0])
for storagedriver in storagedrivers:
if len(storagedriver.vpool.storagedrivers) < 2:
continue
if storagedriver.guid in destination_str.storagedrivers_guids:
if destination_storagedriver is None and (source_storagedriver is None or source_storagedriver.vpool_guid == storagedriver.vpool_guid):
destination_storagedriver = storagedriver
logger.info('Chosen destination storagedriver is: {0}'.format(destination_storagedriver.storage_ip))
elif storagedriver.guid in source_str.storagedrivers_guids:
# Select if the source driver isn't select and destination is also unknown or the storagedriver has matches with the same vpool
if source_storagedriver is None and (destination_storagedriver is None or destination_storagedriver.vpool_guid == storagedriver.vpool_guid):
source_storagedriver = storagedriver
logger.info('Chosen source storagedriver is: {0}'.format(source_storagedriver.storage_ip))
assert source_storagedriver is not None and destination_storagedriver is not None, 'We require at least two storagedrivers within the same domain.'
cluster_info = {'storagerouters': {'destination': destination_str, 'source': source_str, 'compute': compute_str},
'storagedrivers': {'destination': destination_storagedriver, 'source': source_storagedriver}}
compute_client = SSHClient(compute_str, username='******')
is_ee = SystemHelper.get_ovs_version(source_str) == 'ee'
if is_ee is True:
fio_bin_loc = cls.FIO_BIN_EE['location']
fio_bin_url = cls.FIO_BIN_EE['url']
else:
fio_bin_loc = cls.FIO_BIN['location']
fio_bin_url = cls.FIO_BIN['url']
compute_client.run(['wget', fio_bin_url, '-O', fio_bin_loc])
compute_client.file_chmod(fio_bin_loc, 755)
return cluster_info, is_ee, fio_bin_loc
def _get_description():
"""
Retrieve extensive information about the machine
:returns: a extensive description of the local machine
:rtype: str
"""
description_lines = ['# IP INFO']
# fetch ip information
for ip in StoragerouterHelper.get_storagerouter_ips():
description_lines.append('* {0}'.format(ip))
description_lines.append('') # New line gap
# hypervisor information
ci_config = CIConstants.SETUP_CFG
description_lines.append('# HYPERVISOR INFO')
description_lines.append('{0}'.format(
ci_config['ci']['local_hypervisor']['type']))
description_lines.append('') # New line gap
# fetch hardware information
description_lines.append("# HARDWARE INFO")
# board information
description_lines.append("### Base Board Information")
description_lines.append("{0}".format(
subprocess.check_output("dmidecode -t 2",
shell=True).replace("#", "").strip()))
description_lines.append('') # New line gap
# fetch cpu information
description_lines.append("### Processor Information")
output = subprocess.Popen("grep 'model name'",
stdin=subprocess.Popen(
"cat /proc/cpuinfo",
stdout=subprocess.PIPE,
shell=True).stdout,
stdout=subprocess.PIPE,
shell=True)
cpus = subprocess.check_output("cut -d ':' -f 2",
stdin=output.stdout,
shell=True).strip().split('\n')
description_lines.append("* Type: {0}".format(cpus[0]))
description_lines.append("* Amount: {0}".format(len(cpus)))
description_lines.append('') # New line gap
# fetch memory information
description_lines.append("### Memory Information")
output = math.ceil(
float(
subprocess.check_output(
"grep MemTotal",
stdin=subprocess.Popen("cat /proc/meminfo",
stdout=subprocess.PIPE,
shell=True).stdout,
shell=True).strip().split()[1]) / 1024 / 1024)
description_lines.append("* {0}GiB System Memory".format(int(output)))
description_lines.append('') # New line gap
# fetch disk information
description_lines.append("### Disk Information")
output = subprocess.check_output("lsblk", shell=True)
description_lines.append(output.strip())
description_lines.append('') # New line gap
# package info
description_lines.append("# PACKAGE INFO")
description_lines.append("{0}".format(AutoTests._get_package_info()))
return '\n'.join(description_lines)
def validate_add_extend_remove_vpool(cls, timeout=ADD_EXTEND_REMOVE_VPOOL_TIMEOUT):
"""
Validate if we can add, extend and/or remove a vPool, testing the following scenarios:
* Normal with no accelerated backend
* Accelerated vPool with hdd_backend & ssd_backend
INFO:
* at least 2 storagerouters should be available
* at least 2 backends should be available with default preset
:param timeout: specify a timeout
:type timeout: int
:return:
"""
cls.LOGGER.info("Starting to validate add-extend-remove vpool")
storagerouter_ips = []
for storagerouter_ip in StoragerouterHelper.get_storagerouter_ips():
try:
RoleValidation.check_required_roles(VPoolSetup.REQUIRED_VPOOL_ROLES, storagerouter_ip, "LOCAL")
storagerouter_ips.append(storagerouter_ip)
cls.LOGGER.info("Added `{0}` to list of eligible storagerouters".format(storagerouter_ip))
except RuntimeError as ex:
cls.LOGGER.warning("Did not add `{0}` to list of eligible "
"storagerouters because: {1}".format(storagerouter_ip, ex))
pass
# Filter storagerouters without required roles
assert len(storagerouter_ips) > 1, "We need at least 2 storagerouters with valid roles: {0}"\
.format(storagerouter_ips)
alba_backends = BackendHelper.get_alba_backends()
assert len(alba_backends) >= 2, "We need at least 2 or more backends!"
# Global vdisk details
vdisk_deployment_ip = storagerouter_ips[0]
# Determine backends (2)
hdd_backend = alba_backends[0]
ssd_backend = alba_backends[1]
# Add preset to all alba_backends (we only use the first two as seen above)
for alba_backend in alba_backends[0:2]:
cls.LOGGER.info("Adding custom preset to backend {0}".format(alba_backend.name))
preset_result = BackendSetup.add_preset(albabackend_name=alba_backend.name,
preset_details=cls.PRESET,
timeout=cls.PRESET_CREATE_TIMEOUT)
assert preset_result is True, 'Failed to add preset to backend {0}'.format(alba_backend.name)
cls.LOGGER.info("Finished adding custom preset to backend {0}".format(alba_backend.name))
# Vpool configs, regressing https://github.com/openvstorage/alba/issues/560 & more
vpool_configs = {
"no_fragment_cache_on_disk": {
"strategy": {"cache_on_read": False, "cache_on_write": False},
"location": "disk"
},
"no_fragment_cache_on_accel": {
"strategy": {"cache_on_read": False, "cache_on_write": False},
"location": "backend",
"backend": {
"name": ssd_backend.name,
"preset": cls.PRESET['name']
}
}
}
for cfg_name, cfg in vpool_configs.iteritems():
# Create vpool
block_cache_cfg = None
if SystemHelper.get_ovs_version().lower() == 'ee':
block_cache_cfg = cfg
for storagerouter_ip in storagerouter_ips:
cls.LOGGER.info("Add/extend vPool `{0}` on storagerouter `{1}`".format(cls.VPOOL_NAME, storagerouter_ip))
start = time.time()
try:
cls._add_vpool(vpool_name=cls.VPOOL_NAME, fragment_cache_cfg=cfg,
block_cache_cfg=block_cache_cfg, albabackend_name=hdd_backend.name, timeout=timeout,
preset_name=cls.PRESET['name'], storagerouter_ip=storagerouter_ip)
except TimeOutError:
cls.LOGGER.warning('Adding/extending the vpool has timed out after {0}s. Polling for another {1}s.'
.format(timeout, cls.ADD_EXTEND_REMOVE_VPOOL_TIMEOUT_FORGIVING - timeout))
# Lets be a bit forgiving and give the fwk 5 mins to actually complete the task
vpool = VPoolHelper.get_vpool_by_name(cls.VPOOL_NAME)
while vpool.status != 'RUNNING':
if time.time() - start > cls.ADD_EXTEND_REMOVE_VPOOL_TIMEOUT_FORGIVING:
raise RuntimeError('The vpool was not added or extended after {0}s'.format(cls.ADD_EXTEND_REMOVE_VPOOL_TIMEOUT_FORGIVING))
cls.LOGGER.warning('Vpool status is still {0} after {1}s.'.format(vpool.status, time.time() - start))
time.sleep(1)
vpool.discard()
cls.LOGGER.warning('The vpool was added or extended after {0}s.'.format(time.time() - start))
except RuntimeError as ex:
cls.LOGGER.error('Adding/extending the vpool has failed with {0}.'.format(str(ex)))
raise
# Check #proxies
vpool = VPoolHelper.get_vpool_by_name(cls.VPOOL_NAME)
for storagedriver in vpool.storagedrivers:
assert len(storagedriver.alba_proxies) == 2, 'The vpool did not get setup with 2 proxies. Found {} instead.'.format(len(storagedriver.alba_proxies))
# Deploy a vdisk
vdisk_name = cls.PREFIX + cfg_name
cls.LOGGER.info("Starting to create vdisk `{0}` on vPool `{1}` with size `{2}` on node `{3}`"
.format(vdisk_name, cls.VPOOL_NAME, cls.VDISK_SIZE, vdisk_deployment_ip))
VDiskSetup.create_vdisk(vdisk_name=vdisk_name + '.raw',
vpool_name=cls.VPOOL_NAME,
size=cls.VDISK_SIZE,
storagerouter_ip=vdisk_deployment_ip,
timeout=cls.VDISK_CREATE_TIMEOUT)
cls.LOGGER.info("Finished creating vdisk `{0}`".format(vdisk_name))
cls.LOGGER.info("Starting to delete vdisk `{0}`".format(vdisk_name))
VDiskRemover.remove_vdisk_by_name(vdisk_name, cls.VPOOL_NAME)
cls.LOGGER.info("Finished deleting vdisk `{0}`".format(vdisk_name))
# Delete vpool
for storagerouter_ip in storagerouter_ips:
storagedrivers_to_delete = len(vpool.storagedrivers)
cls.LOGGER.info("Deleting vpool `{0}` on storagerouter `{1}`".format(cls.VPOOL_NAME, storagerouter_ip))
try:
VPoolRemover.remove_vpool(vpool_name=cls.VPOOL_NAME, storagerouter_ip=storagerouter_ip, timeout=timeout)
except TimeOutError:
try:
vpool.discard() # Discard is needed to update the vpool status as it was running before
while vpool.status != 'RUNNING':
cls.LOGGER.warning('Removal/shrinking the vpool has timed out after {0}s. Polling for another {1}s.'
.format(timeout, cls.ADD_EXTEND_REMOVE_VPOOL_TIMEOUT_FORGIVING - timeout))
if time.time() - start > cls.ADD_EXTEND_REMOVE_VPOOL_TIMEOUT_FORGIVING:
raise RuntimeError('The vpool was not removed or extended after {0}s'.format(cls.ADD_EXTEND_REMOVE_VPOOL_TIMEOUT_FORGIVING))
cls.LOGGER.warning('Vpool status is still {0} after {1}s.'.format(vpool.status, time.time() - start))
time.sleep(1)
vpool.discard()
except ObjectNotFoundException:
if storagedrivers_to_delete != 1: # Should be last one
raise
except RuntimeError as ex:
cls.LOGGER.error('Shrinking/removing the vpool has failed with {0}.'.format(str(ex)))
raise
cls.LOGGER.info('Vpool has been fully removed.')
# Delete presets
for alba_backend in alba_backends[0:2]:
cls.LOGGER.info("Removing custom preset from backend {0}".format(alba_backend.name))
remove_preset_result = BackendRemover.remove_preset(albabackend_name=alba_backend.name,
preset_name=cls.PRESET['name'],
timeout=cls.PRESET_REMOVE_TIMEOUT)
assert remove_preset_result is True, 'Failed to remove preset from backend {0}'.format(alba_backend.name)
cls.LOGGER.info("Finshed removing custom preset from backend {0}".format(alba_backend.name))
cls.LOGGER.info("Finished to validate add-extend-remove vpool")
def test_collapse():
"""
Test the arakoon collapsing
:return:
"""
ArakoonCollapse.LOGGER.info("Starting validating arakoon collapse")
node_ips = StoragerouterHelper.get_storagerouter_ips()
node_ips.sort()
for node_ip in node_ips:
ArakoonCollapse.LOGGER.info(
"Fetching arakoons on node `{0}`".format(node_ip))
arakoon_clusters = []
root_client = SSHClient(node_ip, username='******')
# fetch arakoon clusters
for service in ServiceList.get_services():
if service.is_internal is True and service.storagerouter.ip == node_ip and \
service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON,
ServiceType.SERVICE_TYPES.NS_MGR,
ServiceType.SERVICE_TYPES.ALBA_MGR):
arakoon_clusters.append(
service.name.replace('arakoon-', ''))
# perform collapse
ArakoonCollapse.LOGGER.info(
"Starting arakoon collapse on node `{0}`".format(node_ip))
for arakoon_cluster in arakoon_clusters:
ArakoonCollapse.LOGGER.info(
"Fetching `{0}` arakoon on node `{1}`".format(
arakoon_cluster, node_ip))
arakoon_config_path = Configuration.get_configuration_path(
'/ovs/arakoon/{0}/config'.format(arakoon_cluster))
tlog_location = '/opt/OpenvStorage/db/arakoon/{0}/tlogs'.format(
arakoon_cluster)
# read_tlog_dir
with remote(node_ip, [Configuration]) as rem:
config_contents = rem.Configuration.get(
'/ovs/arakoon/{0}/config'.format(arakoon_cluster),
raw=True)
for line in config_contents.splitlines():
if 'tlog_dir' in line:
tlog_location = line.split()[-1]
nr_of_tlogs = ArakoonCollapse.get_nr_of_tlogs_in_folder(
root_client, tlog_location)
old_headdb_timestamp = 0
if root_client.file_exists('/'.join([tlog_location,
'head.db'])):
old_headdb_timestamp = root_client.run([
'stat', '--format=%Y',
'{0}/{1}'.format(tlog_location, 'head.db')
])
if nr_of_tlogs <= 2:
benchmark_command = [
'arakoon', '--benchmark', '-n_clients', '1', '-max_n',
'5_000', '-config', arakoon_config_path
]
root_client.run(benchmark_command)
ArakoonCollapse.LOGGER.info(
"Collapsing arakoon `{0}` on node `{1}` ...".format(
arakoon_cluster, node_ip))
GenericController.collapse_arakoon()
nr_of_tlogs = ArakoonCollapse.get_nr_of_tlogs_in_folder(
root_client, tlog_location)
new_headdb_timestamp = root_client.run([
'stat', '--format=%Y',
'{0}/{1}'.format(tlog_location, 'head.db')
])
# perform assertion
assert nr_of_tlogs <= 2,\
'Arakoon collapse left {0} tlogs on the environment, expecting less than 2 in `{1}` on node `{1}`'\
.format(nr_of_tlogs, arakoon_cluster, node_ip)
assert old_headdb_timestamp != new_headdb_timestamp,\
'Timestamp of the head_db file was not changed ' \
'in the process of collapsing tlogs of arakoon `{0}` on node `{1}`'\
.format(arakoon_cluster, node_ip)
ArakoonCollapse.LOGGER.info(
"Successfully collapsed arakoon `{0}` on node `{1}`".
format(arakoon_cluster, node_ip))
ArakoonCollapse.LOGGER.info("Finished validating arakoon collapsing")
def test_archiving(cluster_name='test_archiving',
cluster_basedir='/var/tmp'):
"""
Required method that has to follow our json output guideline
This data will be sent to testrails to process it thereafter
:param cluster_name: name of a non-existing arakoon cluster
:type cluster_name: str
:param cluster_basedir: absolute path for the new arakoon cluster
:type cluster_basedir: str
"""
ArakoonArchiving.LOGGER.info('Starting Arakoon archiving')
storagerouters = StoragerouterHelper.get_storagerouters()
assert len(
storagerouters) >= 2, 'Environment has only `{0}` node(s)'.format(
len(storagerouters))
archived_files = []
files_to_create = []
storagerouters.sort()
for index, storagerouter in enumerate(storagerouters):
# create required directories in cluster_basedir
ovs_client = SSHClient(storagerouter, username='******')
for directory in [
'/'.join([cluster_basedir, 'arakoon']), '/var/log/arakoon'
]:
ovs_client.dir_create(os.path.dirname(directory))
# create required files in cluster_basedir
files_to_create = [
'/'.join(
[cluster_basedir, 'arakoon', cluster_name, 'db',
'one.db']), '/'.join([
cluster_basedir, 'arakoon', cluster_name, 'tlogs',
'one.tlog'
])
]
for filename in files_to_create:
ovs_client.dir_create(os.path.dirname(filename))
ovs_client.file_create(files_to_create)
for filename in files_to_create:
assert ovs_client.file_exists(filename), 'File `{0}` is not present on storagerouter `{1}`'\
.format(filename, storagerouter.ip)
archived_files = [
'/'.join(
['/var/log/arakoon', cluster_name, 'archive', 'one.log'])
]
if index == 0:
ArakoonArchiving.LOGGER.info(
'Starting setup of first arakoon instance of cluster `{0}` '
'on storagerouter `{1}`'.format(cluster_name,
storagerouter.ip))
ArakoonSetup.add_arakoon(cluster_name=cluster_name,
storagerouter_ip=storagerouter.ip,
cluster_basedir=cluster_basedir)
ArakoonArchiving.LOGGER.info(
'Finished setup of first arakoon instance of cluster `{0}`'
'on storagerouter `{1}`'.format(cluster_name,
storagerouter.ip))
else:
ArakoonArchiving.LOGGER.info(
'Starting extending arakoon instance of cluster `{0}` '
'on storagerouter `{1}`'.format(cluster_name,
storagerouter.ip))
ArakoonSetup.extend_arakoon(
cluster_name=cluster_name,
master_storagerouter_ip=storagerouters[0].ip,
storagerouter_ip=storagerouter.ip,
cluster_basedir=cluster_basedir)
ArakoonArchiving.LOGGER.info(
'Finished extending arakoon instance of cluster `{0}` '
'on storagerouter `{1}`'.format(cluster_name,
storagerouter.ip))
ArakoonArchiving.check_archived_directory(ovs_client,
archived_files)
# check required files if they are still present
for filename in files_to_create:
assert ovs_client.file_exists(filename) is False, 'File `{0}` is missing on storagerouter `{1}`'\
.format(filename, storagerouter.ip)
ArakoonArchiving.LOGGER.info(
'Finished test, removing arakoon cluster `{0}`'.format(
cluster_name))
ArakoonRemover.remove_arakoon_cluster(
cluster_name=cluster_name,
master_storagerouter_ip=storagerouters[0].ip)
ArakoonArchiving.LOGGER.info(
'Finished removal of arakoon cluster `{0}`'.format(cluster_name))
# check if required files are removed
for storagerouter in storagerouters:
ovs_client = SSHClient(storagerouter, username='******')
ArakoonArchiving.check_archived_directory(ovs_client,
archived_files)
for filename in files_to_create:
assert ovs_client.file_exists(filename) is False, 'File `{0}` is missing on storagerouter `{1}`'\
.format(filename, storagerouter.ip)
# remove cluster_base_dir
ovs_client.dir_delete("{0}/arakoon".format(cluster_basedir))
def validate_post_reboot(tries=POST_REBOOT_TRIES,
timeout=POST_REBOOT_TIMEOUT):
"""
Validate if all services come up after rebooting a node
:param tries: amount of tries to check if ovs services are running
:type tries: int
:param timeout: timeout between tries
:type timeout: int
:return:
"""
storagerouter_ips = list(StoragerouterHelper.get_storagerouter_ips())
assert len(
storagerouter_ips) >= 2, "We need at least 2 storagerouters!"
PostRebootChecks.LOGGER.info('Starting election of node to reboot')
local_host = SystemHelper.get_local_storagerouter(
).ip # ip address of node where tests are being executed
storagerouter_ips.remove(
local_host
) # remove local ip address so we don't reboot where the tests are running
host_to_reboot = storagerouter_ips[
0] # pick first node that we can find
PostRebootChecks.LOGGER.info(
'Finished election of node to reboot: {0}'.format(host_to_reboot))
# setup beginning ssh connection
client = PostRebootChecks.create_client(host_to_reboot)
# reboot server and wait for it to come up
PostRebootChecks.LOGGER.info(
'Starting reboot of host `{0}`!'.format(host_to_reboot))
client.run(" ( sleep {0} ; reboot ) &".format(
PostRebootChecks.SSH_REBOOT_DELAY))
time.sleep(10)
ssh_tries = 0
while ssh_tries < PostRebootChecks.SSH_WAIT_TRIES:
try:
PostRebootChecks.create_client(host_to_reboot)
PostRebootChecks.LOGGER.info(
'host `{0}` is up again!'.format(host_to_reboot))
break
except Exception:
ssh_tries += 1
PostRebootChecks.LOGGER.warning(
'Host `{0}` still not up at try {1}/{2} ...'.format(
host_to_reboot, ssh_tries,
PostRebootChecks.SSH_WAIT_TRIES))
time.sleep(10) # timeout or else its going too fast
if ssh_tries == PostRebootChecks.SSH_WAIT_TRIES:
# if we reach max tries, throw exception
raise RuntimeError(
"Max amounts of attempts reached ({0}) for host `{1}`, host still not up ..."
.format(ssh_tries, host_to_reboot))
# commence test
PostRebootChecks.LOGGER.info(
'Starting post-reboot service check on node `{0}`'.format(
host_to_reboot))
amount_tries = 0
non_running_services = None
client = SSHClient(host_to_reboot, username='******')
while tries >= amount_tries:
non_running_services = SystemHelper.get_non_running_ovs_services(
client)
if len(non_running_services) == 0:
break
else:
amount_tries += 1
time.sleep(timeout)
assert len(
non_running_services
) == 0, "Found non running services `{0}` after reboot on node `{1}`".format(
non_running_services, host_to_reboot)
PostRebootChecks.LOGGER.info(
'Starting post-reboot vPool check on node `{0}`'.format(
host_to_reboot))
PostRebootChecks.LOGGER.info(
'Finished post-reboot check on node `{0}`'.format(host_to_reboot))