def _vrts_extend_lun(self, volume, size): """Extend vrts LUN to given size.""" lun_name = self._get_va_lun_name(volume.id) target = {'target_name': ''} def _inner(): lun_list = self._get_vrts_lun_list() for lun in lun_list['output']['output']['luns']: if lun['lun_name'] == lun_name: target['target_name'] = lun['target_name'] raise loopingcall.LoopingCallDone() timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_inner) try: timer.start(interval=5, timeout=self.LUN_FOUND_INTERVAL).wait() except loopingcall.LoopingCallTimeOut: return False lun_size = '%sg' % size path = self._lun_extend_str provider = '%s:%s' % (self._va_ip, self._port) data = {} data["lun_name"] = lun_name data["target_name"] = target['target_name'] data["size"] = lun_size result = self._access_api(self.session, provider, path, json.dumps(data), 'POST') return result
def test_looping_call_timed_out(self): def _fake_task(): pass timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_fake_task) self.assertRaises(loopingcall.LoopingCallTimeOut, timer.start(interval=0.1, timeout=0.3).wait)
def run_instance_upgrade(self, expected_states=['UPGRADE', 'HEALTHY'], expected_http_code=202): instance_id = self.instance_info.id self.report.log("Testing upgrade on instance: %s" % instance_id) target_version = self.instance_info.dbaas_datastore_version client = self.auth_client client.instances.upgrade(instance_id, target_version) self.assert_client_code(client, expected_http_code) self.assert_instance_action(instance_id, expected_states) def _wait_for_user_list(): try: all_users = self.get_user_names(client, instance_id) self.report.log("Users in the db instance %s: %s" % (instance_id, all_users)) except Exception as e: self.report.log( "Failed to list users in db instance %s(will continue), " "error: %s" % (instance_id, str(e)) ) else: raise loopingcall.LoopingCallDone() timer = loopingcall.FixedIntervalWithTimeoutLoopingCall( _wait_for_user_list) try: timer.start(interval=3, timeout=120).wait() except loopingcall.LoopingCallTimeOut: self.fail("Timed out: Cannot list users in the db instance %s" % instance_id)
def wait_for_operation(compute, project, operation, interval=1, timeout=60): """Wait for GCE operation to complete, raise error if operation failure :param compute: GCE compute resource object using googleapiclient.discovery :param project: string, GCE Project Id :param zone: string, GCE Name of zone :param operation: object, Operation resource obtained by calling GCE API :param interval: int, Time period(seconds) between two GCE operation checks :param timeout: int, Absoulte time period(seconds) to monitor GCE operation """ def watch_operation(name, request): result = request.execute() if result['status'] == 'DONE': LOG.info("Operation %s status is %s" % (name, result['status'])) if 'error' in result: raise GceOperationError(result['error']) raise loopingcall.LoopingCallDone() operation_name = operation['name'] if 'zone' in operation: zone = operation['zone'].split('/')[-1] monitor_request = compute.zoneOperations().get( project=project, zone=zone, operation=operation_name) elif 'region' in operation: region = operation['region'].split('/')[-1] monitor_request = compute.regionOperations().get( project=project, region=region, operation=operation_name) else: monitor_request = compute.globalOperations().get( project=project, operation=operation_name) timer = loopingcall.FixedIntervalWithTimeoutLoopingCall( watch_operation, operation_name, monitor_request) timer.start(interval=interval, timeout=timeout).wait()
def wait_for_backup_status(cls, id, expected_status=["COMPLETED"], need_delete=False): def _wait(): try: res = cls.client.get_resource("backups", id) cur_status = res["backup"]["status"] except exceptions.NotFound: if need_delete or "DELETED" in expected_status: LOG.info('Backup %s is deleted', id) raise loopingcall.LoopingCallDone() return if cur_status in expected_status: LOG.info('Backup %s becomes %s', id, cur_status) raise loopingcall.LoopingCallDone() elif "FAILED" not in expected_status and cur_status == "FAILED": # If backup status goes to FAILED but is not expected, stop # waiting message = "Backup status is FAILED." caller = test_utils.find_test_caller() if caller: message = '({caller}) {message}'.format(caller=caller, message=message) raise exceptions.UnexpectedResponseCode(message) if type(expected_status) != list: expected_status = [expected_status] if need_delete: # If resource already removed, return try: cls.client.get_resource("backups", id) except exceptions.NotFound: LOG.info('Backup %s not found', id) return LOG.info(f"Deleting backup {id}") cls.delete_backup(id, ignore_notfound=True) timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_wait) try: timer.start(interval=10, timeout=CONF.database.backup_wait_timeout).wait() except loopingcall.LoopingCallTimeOut: message = ("Backup %s is not in the expected status: %s" % (id, expected_status)) caller = test_utils.find_test_caller() if caller: message = '({caller}) {message}'.format(caller=caller, message=message) raise exceptions.TimeoutException(message)
def initialize_connection(self, volume, connector, initiator_data=None): """Initializes the connection and returns connection info. The iscsi driver returns a driver_volume_type of 'iscsi'. the format of the driver data is defined in _vrts_get_iscsi_properties. Example return value:: { 'driver_volume_type': 'iscsi' 'data': { 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_portal': '127.0.0.0.1:3260', 'target_lun': 1, 'volume_id': '12345678-1234-4321-1234-123456789012', } } """ lun_name = self._get_va_lun_name(volume.id) target = {'target_name': ''} def _inner(): lun_list = self._get_vrts_lun_list() for lun in lun_list['output']['output']['luns']: if lun['lun_name'] == lun_name: target['target_name'] = lun['target_name'] raise loopingcall.LoopingCallDone() timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_inner) try: timer.start(interval=5, timeout=self.LUN_FOUND_INTERVAL).wait() except loopingcall.LoopingCallTimeOut: message = (_('ACCESSIscsiDriver initialize_connection ' 'failed for %s as no target was found') % volume.id) LOG.error(message) raise exception.VolumeBackendAPIException(message=message) self._vrts_target_initiator_mapping(target['target_name'], connector['initiator']) iscsi_properties = self._vrts_get_iscsi_properties( volume, target['target_name']) return { 'driver_volume_type': 'iscsi', 'data': iscsi_properties }
def check_server_status(self, server, status): def wait_for_server_status_change(): instance = self.admin_conn.compute.get_server(server.id) if instance.status == status: raise loopingcall.LoopingCallDone() timer = loopingcall.FixedIntervalWithTimeoutLoopingCall( wait_for_server_status_change) try: timer.start(interval=self.SERVER_WAIT_INTERVAL, timeout=self.SERVER_WAIT_PERIOD).wait() except loopingcall.LoopingCallTimeOut: self.fail("Timed out: Instance is not in the expected" " status: %s" % status)
def check_notification_status(self, notification, wait_interval, wait_period): def wait_for_notification_status_finished(): result = self.admin_conn.ha.get_notification( notification.notification_uuid) if result.status == fields.NotificationStatus.FINISHED: raise loopingcall.LoopingCallDone() timer = loopingcall.FixedIntervalWithTimeoutLoopingCall( wait_for_notification_status_finished) try: timer.start(interval=wait_interval, initial_delay=1, timeout=wait_period).wait() except loopingcall.LoopingCallTimeOut: self.fail("Timed out: Notification is not processed and " "it's not in the finished status")
def _move_lun(self, volume, src_ontap_volume, dest_ontap_volume, dest_lun_name=None): """Moves LUN from an ONTAP volume to another.""" job_uuid = self.zapi_client.start_lun_move( volume.name, dest_ontap_volume, src_ontap_volume=src_ontap_volume, dest_lun_name=dest_lun_name) LOG.debug('Start moving LUN %s from %s to %s. ' 'Job UUID is %s.', volume.name, src_ontap_volume, dest_ontap_volume, job_uuid) def _wait_lun_move_complete(): move_status = self.zapi_client.get_lun_move_status(job_uuid) LOG.debug( 'Waiting for LUN move job %s to complete. ' 'Current status is: %s.', job_uuid, move_status['job-status']) if not move_status: status_error_msg = (_("Error moving LUN %s. The " "corresponding Job UUID % doesn't " "exist.")) raise na_utils.NetAppDriverException(status_error_msg % (volume.id, job_uuid)) elif move_status['job-status'] == 'destroyed': status_error_msg = (_('Error moving LUN %s. %s.')) raise na_utils.NetAppDriverException( status_error_msg % (volume.id, move_status['last-failure-reason'])) elif move_status['job-status'] == 'complete': raise loopingcall.LoopingCallDone() try: timer = loopingcall.FixedIntervalWithTimeoutLoopingCall( _wait_lun_move_complete) timer.start(interval=15, timeout=self.configuration. netapp_migrate_volume_timeout).wait() except loopingcall.LoopingCallTimeOut: msg = (_('Timeout waiting to complete move operation of LUN %s.')) raise na_utils.NetAppDriverTimeout(msg % volume.id)
def create_destination_flexvol(self, src_backend_name, dest_backend_name, src_flexvol_name, dest_flexvol_name, pool_is_flexgroup=False): """Create a SnapMirror mirror target FlexVol for a given source.""" dest_backend_config = config_utils.get_backend_configuration( dest_backend_name) dest_vserver = dest_backend_config.netapp_vserver dest_client = config_utils.get_client_for_backend( dest_backend_name, vserver_name=dest_vserver) source_backend_config = config_utils.get_backend_configuration( src_backend_name) src_vserver = source_backend_config.netapp_vserver src_client = config_utils.get_client_for_backend( src_backend_name, vserver_name=src_vserver) provisioning_options = ( src_client.get_provisioning_options_from_flexvol(src_flexvol_name)) provisioning_options.pop('is_flexgroup') # If the source is encrypted then the destination needs to be # encrypted too. Using is_flexvol_encrypted because it includes # a simple check to ensure that the NVE feature is supported. if src_client.is_flexvol_encrypted(src_flexvol_name, src_vserver): provisioning_options['encrypt'] = 'true' # Remove size and volume_type size = provisioning_options.pop('size', None) if not size: msg = _("Unable to read the size of the source FlexVol (%s) " "to create a SnapMirror destination.") raise na_utils.NetAppDriverException(msg % src_flexvol_name) provisioning_options.pop('volume_type', None) source_aggregate = provisioning_options.pop('aggregate') aggregate_map = self._get_replication_aggregate_map( src_backend_name, dest_backend_name) destination_aggregate = [] for src_aggr in source_aggregate: dst_aggr = aggregate_map.get(src_aggr, None) if dst_aggr: destination_aggregate.append(dst_aggr) else: msg = _("Unable to find configuration matching the source " "aggregate and the destination aggregate. Option " "netapp_replication_aggregate_map may be incorrect.") raise na_utils.NetAppDriverException(message=msg) # NOTE(gouthamr): The volume is intentionally created as a Data # Protection volume; junction-path will be added on breaking # the mirror. provisioning_options['volume_type'] = 'dp' if pool_is_flexgroup: compression_enabled = provisioning_options.pop( 'compression_enabled', False) # cDOT compression requires that deduplication be enabled. dedupe_enabled = provisioning_options.pop( 'dedupe_enabled', False) or compression_enabled dest_client.create_volume_async(dest_flexvol_name, destination_aggregate, size, **provisioning_options) timeout = self._get_replication_volume_online_timeout() def _wait_volume_is_online(): volume_state = dest_client.get_volume_state( flexvol_name=dest_flexvol_name) if volume_state and volume_state == 'online': raise loopingcall.LoopingCallDone() try: wait_call = loopingcall.FixedIntervalWithTimeoutLoopingCall( _wait_volume_is_online) wait_call.start(interval=5, timeout=timeout).wait() if dedupe_enabled: dest_client.enable_volume_dedupe_async(dest_flexvol_name) if compression_enabled: dest_client.enable_volume_compression_async( dest_flexvol_name) except loopingcall.LoopingCallTimeOut: msg = _("Timeout waiting destination FlexGroup to to come " "online.") raise na_utils.NetAppDriverException(msg) else: dest_client.create_flexvol(dest_flexvol_name, destination_aggregate[0], size, **provisioning_options)
def _copy_lun(self, volume, src_ontap_volume, src_vserver, dest_ontap_volume, dest_vserver, dest_lun_name=None, dest_backend_name=None, cancel_on_error=False): """Copies LUN from an ONTAP volume to another.""" job_uuid = self.zapi_client.start_lun_copy( volume.name, dest_ontap_volume, dest_vserver, src_ontap_volume=src_ontap_volume, src_vserver=src_vserver, dest_lun_name=dest_lun_name) LOG.debug( 'Start copying LUN %(vol)s from ' '%(src_vserver)s:%(src_ontap_vol)s to ' '%(dest_vserver)s:%(dest_ontap_vol)s. Job UUID is %(job)s.', { 'vol': volume.name, 'src_vserver': src_vserver, 'src_ontap_vol': src_ontap_volume, 'dest_vserver': dest_vserver, 'dest_ontap_vol': dest_ontap_volume, 'job': job_uuid }) def _wait_lun_copy_complete(): copy_status = self.zapi_client.get_lun_copy_status(job_uuid) LOG.debug( 'Waiting for LUN copy job %s to complete. Current ' 'status is: %s.', job_uuid, copy_status['job-status']) if not copy_status: status_error_msg = (_("Error copying LUN %s. The " "corresponding Job UUID % doesn't " "exist.")) raise na_utils.NetAppDriverException(status_error_msg % (volume.id, job_uuid)) elif copy_status['job-status'] == 'destroyed': status_error_msg = (_('Error copying LUN %s. %s.')) raise na_utils.NetAppDriverException( status_error_msg % (volume.id, copy_status['last-failure-reason'])) elif copy_status['job-status'] == 'complete': raise loopingcall.LoopingCallDone() try: timer = loopingcall.FixedIntervalWithTimeoutLoopingCall( _wait_lun_copy_complete) timer.start(interval=10, timeout=self.configuration. netapp_migrate_volume_timeout).wait() except Exception as e: with excutils.save_and_reraise_exception() as ctxt: if cancel_on_error: self._cancel_lun_copy(job_uuid, volume, dest_ontap_volume, dest_backend_name=dest_backend_name) if isinstance(e, loopingcall.LoopingCallTimeOut): ctxt.reraise = False msg = (_('Timeout waiting volume %s to complete ' 'migration.')) raise na_utils.NetAppDriverTimeout(msg % volume.id)
def wait_for_instance_status(cls, id, expected_status=["HEALTHY", "ACTIVE"], need_delete=False, timeout=CONF.database.database_build_timeout): def _wait(): try: res = cls.client.get_resource("instances", id) cur_status = res["instance"]["status"] except exceptions.NotFound: if need_delete or "DELETED" in expected_status: LOG.info('Instance %s is deleted', id) raise loopingcall.LoopingCallDone() return if cur_status in expected_status: LOG.info('Instance %s becomes %s', id, cur_status) raise loopingcall.LoopingCallDone() elif "ERROR" not in expected_status and cur_status == "ERROR": # If instance status goes to ERROR but is not expected, stop # waiting res = cls.admin_client.get_resource("instances", id) LOG.info(f'Instance fault msg: {res["instance"].get("fault")}') # Show trove-guestagent log for debug purpose. # Only admin user is able to publish and show the trove guest # agent log. Make sure the container is deleted after fetching # the log. try: LOG.info(f"Publishing guest log for instance {id}") cls.publish_log(id, 'guest') LOG.info(f"Getting guest log content for instance {id}") log_gen = cls.log_generator(id, 'guest', lines=0) log_content = "".join([chunk for chunk in log_gen()]) LOG.info( f"\n=============================================\n" f"Trove guest agent log for instance {id}\n" f"=============================================") LOG.info(log_content) except Exception as err: LOG.warning(f"Failed to get guest log for instance {id}, " f"error: {str(err)}") finally: # Remove the swift container of database logs. LOG.info(f"Deleting swift container " f"{CONF.database.database_log_container}") cls.delete_swift_containers( cls.admin_container_client, cls.admin_object_client, CONF.database.database_log_container) message = "Instance status is ERROR." caller = test_utils.find_test_caller() if caller: message = '({caller}) {message}'.format(caller=caller, message=message) raise exceptions.UnexpectedResponseCode(message) if type(expected_status) != list: expected_status = [expected_status] if need_delete: # If resource already removed, return try: cls.client.get_resource("instances", id) except exceptions.NotFound: LOG.info('Instance %s not found', id) return LOG.info(f"Deleting instance {id}") cls.admin_client.force_delete_instance(id) timer = loopingcall.FixedIntervalWithTimeoutLoopingCall(_wait) try: timer.start(interval=10, timeout=timeout, initial_delay=5).wait() except loopingcall.LoopingCallTimeOut: message = ("Instance %s is not in the expected status: %s" % (id, expected_status)) caller = test_utils.find_test_caller() if caller: message = '({caller}) {message}'.format(caller=caller, message=message) raise exceptions.TimeoutException(message)