def _vrts_extend_lun(self, volume, size):
"""Extend vrts LUN to given size."""
lun_name = self._get_va_lun_name(volume.id)
target = {'target_name': ''}
def _inner():
lun_list = self._get_vrts_lun_list()
for lun in lun_list['output']['output']['luns']:
if lun['lun_name'] == lun_name:
target['target_name'] = lun['target_name']
raise loopingcall.LoopingCallDone()
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_inner)
try:
timer.start(interval=5, timeout=self.LUN_FOUND_INTERVAL).wait()
except loopingcall.LoopingCallTimeOut:
return False
lun_size = '%sg' % size
path = self._lun_extend_str
provider = '%s:%s' % (self._va_ip, self._port)
data = {}
data["lun_name"] = lun_name
data["target_name"] = target['target_name']
data["size"] = lun_size
result = self._access_api(self.session, provider, path,
json.dumps(data), 'POST')
return result
def test_looping_call_timed_out(self):
def _fake_task():
pass
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_fake_task)
self.assertRaises(loopingcall.LoopingCallTimeOut,
timer.start(interval=0.1, timeout=0.3).wait)
def run_instance_upgrade(self, expected_states=['UPGRADE', 'HEALTHY'],
expected_http_code=202):
instance_id = self.instance_info.id
self.report.log("Testing upgrade on instance: %s" % instance_id)
target_version = self.instance_info.dbaas_datastore_version
client = self.auth_client
client.instances.upgrade(instance_id, target_version)
self.assert_client_code(client, expected_http_code)
self.assert_instance_action(instance_id, expected_states)
def _wait_for_user_list():
try:
all_users = self.get_user_names(client, instance_id)
self.report.log("Users in the db instance %s: %s" %
(instance_id, all_users))
except Exception as e:
self.report.log(
"Failed to list users in db instance %s(will continue), "
"error: %s" % (instance_id, str(e))
)
else:
raise loopingcall.LoopingCallDone()
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_for_user_list)
try:
timer.start(interval=3, timeout=120).wait()
except loopingcall.LoopingCallTimeOut:
self.fail("Timed out: Cannot list users in the db instance %s"
% instance_id)
def wait_for_operation(compute, project, operation, interval=1, timeout=60):
"""Wait for GCE operation to complete, raise error if operation failure
:param compute: GCE compute resource object using googleapiclient.discovery
:param project: string, GCE Project Id
:param zone: string, GCE Name of zone
:param operation: object, Operation resource obtained by calling GCE API
:param interval: int, Time period(seconds) between two GCE operation checks
:param timeout: int, Absoulte time period(seconds) to monitor GCE operation
"""
def watch_operation(name, request):
result = request.execute()
if result['status'] == 'DONE':
LOG.info("Operation %s status is %s" % (name, result['status']))
if 'error' in result:
raise GceOperationError(result['error'])
raise loopingcall.LoopingCallDone()
operation_name = operation['name']
if 'zone' in operation:
zone = operation['zone'].split('/')[-1]
monitor_request = compute.zoneOperations().get(
project=project, zone=zone, operation=operation_name)
elif 'region' in operation:
region = operation['region'].split('/')[-1]
monitor_request = compute.regionOperations().get(
project=project, region=region, operation=operation_name)
else:
monitor_request = compute.globalOperations().get(
project=project, operation=operation_name)
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
watch_operation, operation_name, monitor_request)
timer.start(interval=interval, timeout=timeout).wait()
def wait_for_backup_status(cls, id, expected_status=["COMPLETED"],
need_delete=False):
def _wait():
try:
res = cls.client.get_resource("backups", id)
cur_status = res["backup"]["status"]
except exceptions.NotFound:
if need_delete or "DELETED" in expected_status:
LOG.info('Backup %s is deleted', id)
raise loopingcall.LoopingCallDone()
return
if cur_status in expected_status:
LOG.info('Backup %s becomes %s', id, cur_status)
raise loopingcall.LoopingCallDone()
elif "FAILED" not in expected_status and cur_status == "FAILED":
# If backup status goes to FAILED but is not expected, stop
# waiting
message = "Backup status is FAILED."
caller = test_utils.find_test_caller()
if caller:
message = '({caller}) {message}'.format(caller=caller,
message=message)
raise exceptions.UnexpectedResponseCode(message)
if type(expected_status) != list:
expected_status = [expected_status]
if need_delete:
# If resource already removed, return
try:
cls.client.get_resource("backups", id)
except exceptions.NotFound:
LOG.info('Backup %s not found', id)
return
LOG.info(f"Deleting backup {id}")
cls.delete_backup(id, ignore_notfound=True)
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_wait)
try:
timer.start(interval=10,
timeout=CONF.database.backup_wait_timeout).wait()
except loopingcall.LoopingCallTimeOut:
message = ("Backup %s is not in the expected status: %s" %
(id, expected_status))
caller = test_utils.find_test_caller()
if caller:
message = '({caller}) {message}'.format(caller=caller,
message=message)
raise exceptions.TimeoutException(message)
def initialize_connection(self, volume, connector, initiator_data=None):
"""Initializes the connection and returns connection info.
The iscsi driver returns a driver_volume_type of 'iscsi'.
the format of the driver data is defined in _vrts_get_iscsi_properties.
Example return value::
{
'driver_volume_type': 'iscsi'
'data': {
'target_discovered': True,
'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',
'target_portal': '127.0.0.0.1:3260',
'target_lun': 1,
'volume_id': '12345678-1234-4321-1234-123456789012',
}
}
"""
lun_name = self._get_va_lun_name(volume.id)
target = {'target_name': ''}
def _inner():
lun_list = self._get_vrts_lun_list()
for lun in lun_list['output']['output']['luns']:
if lun['lun_name'] == lun_name:
target['target_name'] = lun['target_name']
raise loopingcall.LoopingCallDone()
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_inner)
try:
timer.start(interval=5, timeout=self.LUN_FOUND_INTERVAL).wait()
except loopingcall.LoopingCallTimeOut:
message = (_('ACCESSIscsiDriver initialize_connection '
'failed for %s as no target was found')
% volume.id)
LOG.error(message)
raise exception.VolumeBackendAPIException(message=message)
self._vrts_target_initiator_mapping(target['target_name'],
connector['initiator'])
iscsi_properties = self._vrts_get_iscsi_properties(
volume, target['target_name'])
return {
'driver_volume_type': 'iscsi',
'data': iscsi_properties
}
def check_server_status(self, server, status):
def wait_for_server_status_change():
instance = self.admin_conn.compute.get_server(server.id)
if instance.status == status:
raise loopingcall.LoopingCallDone()
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
wait_for_server_status_change)
try:
timer.start(interval=self.SERVER_WAIT_INTERVAL,
timeout=self.SERVER_WAIT_PERIOD).wait()
except loopingcall.LoopingCallTimeOut:
self.fail("Timed out: Instance is not in the expected"
" status: %s" % status)
def check_notification_status(self, notification, wait_interval,
wait_period):
def wait_for_notification_status_finished():
result = self.admin_conn.ha.get_notification(
notification.notification_uuid)
if result.status == fields.NotificationStatus.FINISHED:
raise loopingcall.LoopingCallDone()
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
wait_for_notification_status_finished)
try:
timer.start(interval=wait_interval,
initial_delay=1,
timeout=wait_period).wait()
except loopingcall.LoopingCallTimeOut:
self.fail("Timed out: Notification is not processed and "
"it's not in the finished status")
def _move_lun(self,
volume,
src_ontap_volume,
dest_ontap_volume,
dest_lun_name=None):
"""Moves LUN from an ONTAP volume to another."""
job_uuid = self.zapi_client.start_lun_move(
volume.name,
dest_ontap_volume,
src_ontap_volume=src_ontap_volume,
dest_lun_name=dest_lun_name)
LOG.debug('Start moving LUN %s from %s to %s. '
'Job UUID is %s.', volume.name, src_ontap_volume,
dest_ontap_volume, job_uuid)
def _wait_lun_move_complete():
move_status = self.zapi_client.get_lun_move_status(job_uuid)
LOG.debug(
'Waiting for LUN move job %s to complete. '
'Current status is: %s.', job_uuid, move_status['job-status'])
if not move_status:
status_error_msg = (_("Error moving LUN %s. The "
"corresponding Job UUID % doesn't "
"exist."))
raise na_utils.NetAppDriverException(status_error_msg %
(volume.id, job_uuid))
elif move_status['job-status'] == 'destroyed':
status_error_msg = (_('Error moving LUN %s. %s.'))
raise na_utils.NetAppDriverException(
status_error_msg %
(volume.id, move_status['last-failure-reason']))
elif move_status['job-status'] == 'complete':
raise loopingcall.LoopingCallDone()
try:
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_lun_move_complete)
timer.start(interval=15,
timeout=self.configuration.
netapp_migrate_volume_timeout).wait()
except loopingcall.LoopingCallTimeOut:
msg = (_('Timeout waiting to complete move operation of LUN %s.'))
raise na_utils.NetAppDriverTimeout(msg % volume.id)
def create_destination_flexvol(self,
src_backend_name,
dest_backend_name,
src_flexvol_name,
dest_flexvol_name,
pool_is_flexgroup=False):
"""Create a SnapMirror mirror target FlexVol for a given source."""
dest_backend_config = config_utils.get_backend_configuration(
dest_backend_name)
dest_vserver = dest_backend_config.netapp_vserver
dest_client = config_utils.get_client_for_backend(
dest_backend_name, vserver_name=dest_vserver)
source_backend_config = config_utils.get_backend_configuration(
src_backend_name)
src_vserver = source_backend_config.netapp_vserver
src_client = config_utils.get_client_for_backend(
src_backend_name, vserver_name=src_vserver)
provisioning_options = (
src_client.get_provisioning_options_from_flexvol(src_flexvol_name))
provisioning_options.pop('is_flexgroup')
# If the source is encrypted then the destination needs to be
# encrypted too. Using is_flexvol_encrypted because it includes
# a simple check to ensure that the NVE feature is supported.
if src_client.is_flexvol_encrypted(src_flexvol_name, src_vserver):
provisioning_options['encrypt'] = 'true'
# Remove size and volume_type
size = provisioning_options.pop('size', None)
if not size:
msg = _("Unable to read the size of the source FlexVol (%s) "
"to create a SnapMirror destination.")
raise na_utils.NetAppDriverException(msg % src_flexvol_name)
provisioning_options.pop('volume_type', None)
source_aggregate = provisioning_options.pop('aggregate')
aggregate_map = self._get_replication_aggregate_map(
src_backend_name, dest_backend_name)
destination_aggregate = []
for src_aggr in source_aggregate:
dst_aggr = aggregate_map.get(src_aggr, None)
if dst_aggr:
destination_aggregate.append(dst_aggr)
else:
msg = _("Unable to find configuration matching the source "
"aggregate and the destination aggregate. Option "
"netapp_replication_aggregate_map may be incorrect.")
raise na_utils.NetAppDriverException(message=msg)
# NOTE(gouthamr): The volume is intentionally created as a Data
# Protection volume; junction-path will be added on breaking
# the mirror.
provisioning_options['volume_type'] = 'dp'
if pool_is_flexgroup:
compression_enabled = provisioning_options.pop(
'compression_enabled', False)
# cDOT compression requires that deduplication be enabled.
dedupe_enabled = provisioning_options.pop(
'dedupe_enabled', False) or compression_enabled
dest_client.create_volume_async(dest_flexvol_name,
destination_aggregate, size,
**provisioning_options)
timeout = self._get_replication_volume_online_timeout()
def _wait_volume_is_online():
volume_state = dest_client.get_volume_state(
flexvol_name=dest_flexvol_name)
if volume_state and volume_state == 'online':
raise loopingcall.LoopingCallDone()
try:
wait_call = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_volume_is_online)
wait_call.start(interval=5, timeout=timeout).wait()
if dedupe_enabled:
dest_client.enable_volume_dedupe_async(dest_flexvol_name)
if compression_enabled:
dest_client.enable_volume_compression_async(
dest_flexvol_name)
except loopingcall.LoopingCallTimeOut:
msg = _("Timeout waiting destination FlexGroup to to come "
"online.")
raise na_utils.NetAppDriverException(msg)
else:
dest_client.create_flexvol(dest_flexvol_name,
destination_aggregate[0], size,
**provisioning_options)
def _copy_lun(self,
volume,
src_ontap_volume,
src_vserver,
dest_ontap_volume,
dest_vserver,
dest_lun_name=None,
dest_backend_name=None,
cancel_on_error=False):
"""Copies LUN from an ONTAP volume to another."""
job_uuid = self.zapi_client.start_lun_copy(
volume.name,
dest_ontap_volume,
dest_vserver,
src_ontap_volume=src_ontap_volume,
src_vserver=src_vserver,
dest_lun_name=dest_lun_name)
LOG.debug(
'Start copying LUN %(vol)s from '
'%(src_vserver)s:%(src_ontap_vol)s to '
'%(dest_vserver)s:%(dest_ontap_vol)s. Job UUID is %(job)s.', {
'vol': volume.name,
'src_vserver': src_vserver,
'src_ontap_vol': src_ontap_volume,
'dest_vserver': dest_vserver,
'dest_ontap_vol': dest_ontap_volume,
'job': job_uuid
})
def _wait_lun_copy_complete():
copy_status = self.zapi_client.get_lun_copy_status(job_uuid)
LOG.debug(
'Waiting for LUN copy job %s to complete. Current '
'status is: %s.', job_uuid, copy_status['job-status'])
if not copy_status:
status_error_msg = (_("Error copying LUN %s. The "
"corresponding Job UUID % doesn't "
"exist."))
raise na_utils.NetAppDriverException(status_error_msg %
(volume.id, job_uuid))
elif copy_status['job-status'] == 'destroyed':
status_error_msg = (_('Error copying LUN %s. %s.'))
raise na_utils.NetAppDriverException(
status_error_msg %
(volume.id, copy_status['last-failure-reason']))
elif copy_status['job-status'] == 'complete':
raise loopingcall.LoopingCallDone()
try:
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(
_wait_lun_copy_complete)
timer.start(interval=10,
timeout=self.configuration.
netapp_migrate_volume_timeout).wait()
except Exception as e:
with excutils.save_and_reraise_exception() as ctxt:
if cancel_on_error:
self._cancel_lun_copy(job_uuid,
volume,
dest_ontap_volume,
dest_backend_name=dest_backend_name)
if isinstance(e, loopingcall.LoopingCallTimeOut):
ctxt.reraise = False
msg = (_('Timeout waiting volume %s to complete '
'migration.'))
raise na_utils.NetAppDriverTimeout(msg % volume.id)
def wait_for_instance_status(cls, id,
expected_status=["HEALTHY", "ACTIVE"],
need_delete=False,
timeout=CONF.database.database_build_timeout):
def _wait():
try:
res = cls.client.get_resource("instances", id)
cur_status = res["instance"]["status"]
except exceptions.NotFound:
if need_delete or "DELETED" in expected_status:
LOG.info('Instance %s is deleted', id)
raise loopingcall.LoopingCallDone()
return
if cur_status in expected_status:
LOG.info('Instance %s becomes %s', id, cur_status)
raise loopingcall.LoopingCallDone()
elif "ERROR" not in expected_status and cur_status == "ERROR":
# If instance status goes to ERROR but is not expected, stop
# waiting
res = cls.admin_client.get_resource("instances", id)
LOG.info(f'Instance fault msg: {res["instance"].get("fault")}')
# Show trove-guestagent log for debug purpose.
# Only admin user is able to publish and show the trove guest
# agent log. Make sure the container is deleted after fetching
# the log.
try:
LOG.info(f"Publishing guest log for instance {id}")
cls.publish_log(id, 'guest')
LOG.info(f"Getting guest log content for instance {id}")
log_gen = cls.log_generator(id, 'guest', lines=0)
log_content = "".join([chunk for chunk in log_gen()])
LOG.info(
f"\n=============================================\n"
f"Trove guest agent log for instance {id}\n"
f"=============================================")
LOG.info(log_content)
except Exception as err:
LOG.warning(f"Failed to get guest log for instance {id}, "
f"error: {str(err)}")
finally:
# Remove the swift container of database logs.
LOG.info(f"Deleting swift container "
f"{CONF.database.database_log_container}")
cls.delete_swift_containers(
cls.admin_container_client, cls.admin_object_client,
CONF.database.database_log_container)
message = "Instance status is ERROR."
caller = test_utils.find_test_caller()
if caller:
message = '({caller}) {message}'.format(caller=caller,
message=message)
raise exceptions.UnexpectedResponseCode(message)
if type(expected_status) != list:
expected_status = [expected_status]
if need_delete:
# If resource already removed, return
try:
cls.client.get_resource("instances", id)
except exceptions.NotFound:
LOG.info('Instance %s not found', id)
return
LOG.info(f"Deleting instance {id}")
cls.admin_client.force_delete_instance(id)
timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_wait)
try:
timer.start(interval=10, timeout=timeout, initial_delay=5).wait()
except loopingcall.LoopingCallTimeOut:
message = ("Instance %s is not in the expected status: %s" %
(id, expected_status))
caller = test_utils.find_test_caller()
if caller:
message = '({caller}) {message}'.format(caller=caller,
message=message)
raise exceptions.TimeoutException(message)
