def _connect_target_volume(self, target_nqn, vol_uuid, portals):
try:
host_device_path = NVMeOFConnector.get_nvme_device_path(
self, target_nqn, vol_uuid)
except exception.VolumeDeviceNotFound:
host_device_path = None
if not host_device_path:
any_connect = NVMeOFConnector.connect_to_portals(
self, target_nqn, portals)
if not any_connect:
LOG.error("No successful connections: %(host_devices)s",
{'host_devices': target_nqn})
raise exception.VolumeDeviceNotFound(device=target_nqn)
host_device_path = NVMeOFConnector.get_nvme_device_path(
self, target_nqn, vol_uuid)
if not host_device_path:
LOG.error("No accessible volume device: %(host_devices)s",
{'host_devices': target_nqn})
raise exception.VolumeDeviceNotFound(device=target_nqn)
else:
NVMeOFConnector.rescan(self, target_nqn, vol_uuid)
host_device_path = NVMeOFConnector.get_nvme_device_path(
self, target_nqn, vol_uuid)
return host_device_path
def create_lv_snapshot(self, name, source_lv_name, lv_type='default'):
"""Creates a snapshot of a logical volume.
:param name: Name to assign to new snapshot
:param source_lv_name: Name of Logical Volume to snapshot
:param lv_type: Type of LV (default or thin)
"""
source_lvref = self.get_volume(source_lv_name)
if source_lvref is None:
LOG.error("Trying to create snapshot by non-existent LV: %s",
source_lv_name)
raise exception.VolumeDeviceNotFound(device=source_lv_name)
cmd = LVM.LVM_CMD_PREFIX + [
'lvcreate', '--name', name, '-k', 'y', '--snapshot',
'%s/%s' % (self.vg_name, source_lv_name)
]
if lv_type != 'thin':
size = source_lvref['size']
cmd.extend(['-L', '%sg' % (size)])
try:
self._execute(*cmd,
root_helper=self._root_helper,
run_as_root=True)
except putils.ProcessExecutionError as err:
LOG.exception('Error creating snapshot')
LOG.error('Cmd :%s', err.cmd)
LOG.error('StdOut :%s', err.stdout)
LOG.error('StdErr :%s', err.stderr)
raise
def _check_rbd_device():
rbd_dev_path = self.get_device_name(connection_properties,
expect=False)
if rbd_dev_path:
try:
# Under high load, it can take a second before the disk
# becomes accessible.
with open(rbd_dev_path, 'rb'):
pass
nonlocal dev_path
dev_path = rbd_dev_path
raise loopingcall.LoopingCallDone()
except FileNotFoundError:
LOG.debug(
"The RBD image %(image)s mapped to local device "
"%(dev)s isn't available yet.", {
'image': connection_properties['name'],
'dev': rbd_dev_path
})
nonlocal attempt
attempt += 1
if attempt >= self.device_scan_attempts:
msg = _("The mounted RBD image isn't available: %s")
raise exception.VolumeDeviceNotFound(
msg % connection_properties['name'])
def _connect_single_volume(self, connection_properties):
"""Connect to a volume using a single path."""
data = {'stop_connecting': False, 'num_logins': 0, 'failed_logins': 0,
'stopped_threads': 0, 'found_devices': [],
'just_added_devices': []}
for props in self._iterate_all_targets(connection_properties):
self._connect_vol(self.device_scan_attempts, props, data)
found_devs = data['found_devices']
if found_devs:
for __ in range(10):
wwn = self._linuxscsi.get_sysfs_wwn(found_devs)
if wwn:
return self._get_connect_result(connection_properties,
wwn, found_devs)
time.sleep(1)
LOG.debug('Could not find the WWN for %s.', found_devs[0])
# If we failed we must cleanup the connection, as we could be
# leaving the node entry if it's not being used by another device.
ips_iqns_luns = ((props['target_portal'], props['target_iqn'],
props['target_lun']), )
self._cleanup_connection(props, ips_iqns_luns, force=True,
ignore_errors=True)
# Reset connection result values for next try
data.update(num_logins=0, failed_logins=0, found_devices=[])
raise exception.VolumeDeviceNotFound(device='')
def wait_for_path(self, volume_path):
"""Wait for a path to show up."""
LOG.debug("Checking to see if %s exists yet.", volume_path)
if not os.path.exists(volume_path):
LOG.debug("%(path)s doesn't exists yet.", {'path': volume_path})
raise exception.VolumeDeviceNotFound(device=volume_path)
else:
LOG.debug("%s has shown up.", volume_path)
def get_device_name(self, connection_properties, expect=True):
mapping = self._show_rbd_mapping(connection_properties)
if mapping:
dev_num = mapping['disk_number']
return self._diskutils.get_device_name_by_device_number(dev_num)
elif expect:
msg = _("The specified RBD image is not mounted: %s")
raise exception.VolumeDeviceNotFound(msg %
connection_properties['name'])
def test__connect_target_volume_not_connected(self, mock_device_path,
mock_portals):
mock_device_path.side_effect = exception.VolumeDeviceNotFound()
mock_portals.return_value = True
self.assertRaises(exception.VolumeDeviceNotFound,
self.connector._connect_target_volume, TARGET_NQN,
VOL_UUID, [('fake', 'portal', 'tcp')])
mock_device_path.assert_called_with(self.connector, TARGET_NQN,
VOL_UUID)
def _wait_for_discovery(aoe_path):
if os.path.exists(aoe_path):
raise loopingcall.LoopingCallDone
if waiting_status['tries'] >= self.device_scan_attempts:
raise exception.VolumeDeviceNotFound(device=aoe_path)
LOG.info("AoE volume not yet found at: %(path)s. "
"Try number: %(tries)s",
{'path': aoe_device, 'tries': waiting_status['tries']})
self._aoe_discover()
waiting_status['tries'] += 1
def test_libvirt_iscsi_driver_disconnect_volume_with_devicenotfound(self,
mock_LOG_warning):
device_path = '/dev/fake-dev'
connection_info = {'data': {'device_path': device_path}}
libvirt_driver = iscsi.LibvirtISCSIVolumeDriver(self.fake_conn)
libvirt_driver.connector.disconnect_volume = mock.MagicMock(
side_effect=os_brick_exception.VolumeDeviceNotFound(
device=device_path))
libvirt_driver.disconnect_volume(connection_info, device_path)
msg = mock_LOG_warning.call_args_list[0]
self.assertIn('Ignoring VolumeDeviceNotFound', msg[0][0])
def _get_nvme_controller(executor, target_nqn):
ctrls = glob.glob('/sys/class/nvme-fabrics/ctl/nvme*')
for ctrl in ctrls:
try:
lines, _err = executor._execute(
'cat',
ctrl + '/subsysnqn',
run_as_root=True,
root_helper=executor._root_helper)
for line in lines.split('\n'):
if line == target_nqn:
state, _err = executor._execute(
'cat',
ctrl + '/state',
run_as_root=True,
root_helper=executor._root_helper)
if 'live' not in state:
LOG.debug("nvmeof ctrl device not live: %s", ctrl)
raise exception.VolumeDeviceNotFound(device=ctrl)
return ctrl[ctrl.rfind('/') + 1:]
except putils.ProcessExecutionError as e:
LOG.exception(e)
raise exception.VolumeDeviceNotFound(device=target_nqn)
def get_nvme_device_path(executor, target_nqn, vol_uuid):
nvme_ctrl = NVMeOFConnector._get_nvme_controller(executor, target_nqn)
try:
blocks = glob.glob('/sys/class/nvme-fabrics/ctl/' + nvme_ctrl +
'/' + nvme_ctrl + 'n*')
for block in blocks:
uuid_lines, _err = executor._execute(
'cat',
block + '/uuid',
run_as_root=True,
root_helper=executor._root_helper)
if uuid_lines.split('\n')[0] == vol_uuid:
return '/dev/' + block[block.rfind('/') + 1:]
except putils.ProcessExecutionError as e:
LOG.exception(e)
raise exception.VolumeDeviceNotFound(device=vol_uuid)
def disconnect_volume(self, connection_properties, device_info):
"""Detach the volume from instance_name.
:param connection_properties: The dictionary that describes all
of the target volume attributes.
:type connection_properties: dict
:param device_info: historical difference, but same as connection_props
:type device_info: dict
connection_properties for iSCSI must include:
target_portal(s) - IP and optional port
target_iqn(s) - iSCSI Qualified Name
target_lun(s) - LUN id of the volume
"""
if self.use_multipath:
self._rescan_multipath()
host_device = multipath_device = None
host_devices = self._get_device_path(connection_properties)
# Choose an accessible host device
for dev in host_devices:
if os.path.exists(dev):
host_device = dev
device_wwn = self._linuxscsi.get_scsi_wwn(dev)
(multipath_device, multipath_id) = (super(
ISCSIConnector,
self)._discover_mpath_device(device_wwn,
connection_properties,
dev))
if multipath_device:
break
if not host_device:
LOG.error(_LE("No accessible volume device: %(host_devices)s"),
{'host_devices': host_devices})
raise exception.VolumeDeviceNotFound(device=host_devices)
if multipath_device:
device_realpath = os.path.realpath(host_device)
self._linuxscsi.remove_multipath_device(device_realpath)
return self._disconnect_volume_multipath_iscsi(
connection_properties, multipath_device)
# When multiple portals/iqns/luns are specified, we need to remove
# unused devices created by logging into other LUNs' session.
for props in self._iterate_all_targets(connection_properties):
self._disconnect_volume_iscsi(props)
def _connect_volume_replicated(self, connection_properties):
"""connect to volume on host
connection_properties for NVMe-oF must include:
target_portals - list of ip,port,transport for each portal
target_nqn - NVMe-oF Qualified Name
vol_uuid - UUID for volume/replica
"""
volume_replicas = connection_properties.get('volume_replicas')
volume_alias = connection_properties.get('alias')
if volume_replicas:
host_device_paths = []
for replica in volume_replicas:
try:
rep_host_device_path = self._connect_target_volume(
replica['target_nqn'], replica['vol_uuid'],
replica['portals'])
if rep_host_device_path:
host_device_paths.append(rep_host_device_path)
except Exception as ex:
LOG.error("_connect_target_volume: %s", ex)
if not host_device_paths:
raise exception.VolumeDeviceNotFound(device=volume_replicas)
if len(volume_replicas) > 1:
device_path = self._handle_replicated_volume(
host_device_paths, volume_alias, len(volume_replicas))
else:
device_path = self._handle_single_replica(
host_device_paths, volume_alias)
else:
device_path = self._connect_target_volume(
connection_properties['target_nqn'],
connection_properties['vol_uuid'],
connection_properties['portals'])
if nvmeof_agent:
nvmeof_agent.NVMeOFAgent.ensure_running(self)
return {'type': 'block', 'path': device_path}
def _get_device_link(self, wwn, device, mpath):
# These are the default symlinks that should always be there
if mpath:
symlink = '/dev/disk/by-id/dm-uuid-mpath-' + mpath
else:
symlink = '/dev/disk/by-id/scsi-' + wwn
# If default symlinks are not there just search for anything that links
# to our device. In my experience this will return the last added link
# first, so if we are going to succeed this should be fast.
if not os.path.realpath(symlink) == device:
links_path = '/dev/disk/by-id/'
for symlink in os.listdir(links_path):
symlink = links_path + symlink
if os.path.realpath(symlink) == device:
break
else:
# Raising this will trigger the next retry
raise exception.VolumeDeviceNotFound(device='/dev/disk/by-id')
return symlink
def _handle_replicated_volume(self, host_device_paths, volume_alias,
num_of_replicas):
path_in_raid = False
for dev_path in host_device_paths:
path_in_raid = NVMeOFConnector._is_device_in_raid(self, dev_path)
if path_in_raid:
break
device_path = '/dev/md/' + volume_alias
if path_in_raid:
NVMeOFConnector.stop_and_assemble_raid(self, host_device_paths,
device_path, False)
else:
paths_found = len(host_device_paths)
if num_of_replicas > paths_found:
LOG.error('Cannot create MD as %s out of %s legs were found.',
paths_found, num_of_replicas)
raise exception.VolumeDeviceNotFound(device=volume_alias)
NVMeOFConnector.create_raid(self, host_device_paths, '1',
volume_alias, volume_alias, False)
return device_path
def connect_volume(self, connection_properties, scan_tries):
"""Attach the volume to instance_name.
connection_properties for iSCSI must include:
target_portal - ip and optional port
target_iqn - iSCSI Qualified Name
target_lun - LUN id of the volume
"""
device_info = {'type': 'block'}
# TODO(Strony): support the iSCSI multipath on Solaris.
self._connect_to_iscsi_portal(connection_properties)
host_device = self._get_device_path(connection_properties)
# check if it is a valid device path.
for i in range(1, scan_tries):
if os.path.exists(host_device):
break
else:
time.sleep(i**2)
else:
raise exception.VolumeDeviceNotFound(device=host_device)
# Set the label EFI to the disk on SPARC before it is accessed and
# make sure the correct device path with slice 0
# (like '/dev/rdsk/c0t600xxxd0s0').
if platform.processor() == 'sparc':
tmp_dev_name = host_device.rsplit('s', 1)
disk_name = tmp_dev_name[0].split('/')[-1]
(out, _err) = self.execute('/usr/sbin/format', '-L', 'efi', '-d',
disk_name)
host_device = '%ss0' % tmp_dev_name[0]
device_info['path'] = host_device
return device_info
def always_fails():
self.counter += 1
raise exception.VolumeDeviceNotFound(device='fake')
def connect_volume(self, connection_properties):
"""Attach the volume to instance_name.
NOTE: Will retry up to three times to handle the case where c-vol
and n-cpu are both using os-brick to manage iSCSI sessions but they
are on the same node and using different locking directories. In this
case, even though this call is synchronized, they will be separate
locks and can still overlap with connect and disconnect. Since a
disconnect during an initial attach can't cause IO failure (the device
has not been made available yet), we just try the connection again.
:param connection_properties: The valid dictionary that describes all
of the target volume attributes.
:type connection_properties: dict
:returns: dict
connection_properties for iSCSI must include:
target_portal(s) - ip and optional port
target_iqn(s) - iSCSI Qualified Name
target_lun(s) - LUN id of the volume
Note that plural keys may be used when use_multipath=True
"""
device_info = {'type': 'block'}
# At this point the host_devices may be an empty list
host_devices, target_props = self._get_potential_volume_paths(
connection_properties)
# The /dev/disk/by-path/... node is not always present immediately
# TODO(justinsb): This retry-with-delay is a pattern, move to utils?
tries = 0
# Loop until at least 1 path becomes available
while all(map(lambda x: not os.path.exists(x), host_devices)):
if tries >= self.device_scan_attempts:
raise exception.VolumeDeviceNotFound(device=host_devices)
LOG.info(
"ISCSI volume not yet found at: %(host_devices)s. "
"Will rescan & retry. Try number: %(tries)s.", {
'host_devices': host_devices,
'tries': tries
})
if self.use_multipath:
# We need to refresh the paths as the devices may be empty
host_devices, target_props = (
self._get_potential_volume_paths(connection_properties))
else:
if (tries):
host_devices = self._get_device_path(target_props)
self._run_iscsiadm(target_props, ("--rescan", ))
tries = tries + 1
if all(map(lambda x: not os.path.exists(x), host_devices)):
time.sleep(tries**2)
else:
break
if tries != 0:
LOG.debug(
"Found iSCSI node %(host_devices)s "
"(after %(tries)s rescans)", {
'host_devices': host_devices,
'tries': tries
})
# Choose an accessible host device
host_device = next(dev for dev in host_devices if os.path.exists(dev))
# find out the WWN of the device
device_wwn = self._linuxscsi.get_scsi_wwn(host_device)
LOG.debug("Device WWN = '%(wwn)s'", {'wwn': device_wwn})
device_info['scsi_wwn'] = device_wwn
if self.use_multipath:
(host_device, multipath_id) = (super(ISCSIConnector,
self)._discover_mpath_device(
device_wwn,
connection_properties,
host_device))
if multipath_id:
device_info['multipath_id'] = multipath_id
device_info['path'] = host_device
LOG.debug("connect_volume returning %s", device_info)
return device_info
def connect_volume(self, connection_properties):
"""Attach the volume to instance_name.
:param connection_properties: The valid dictionary that describes all
of the target volume attributes.
:type connection_properties: dict
:returns: dict
connection_properties for iSCSI must include:
target_portal(s) - ip and optional port
target_iqn(s) - iSCSI Qualified Name
target_lun(s) - LUN id of the volume
Note that plural keys may be used when use_multipath=True
"""
device_info = {'type': 'block'}
# At this point the host_devices may be an empty list
host_devices, target_props = self._get_potential_volume_paths(
connection_properties)
# The /dev/disk/by-path/... node is not always present immediately
# TODO(justinsb): This retry-with-delay is a pattern, move to utils?
tries = 0
# Loop until at least 1 path becomes available
while all(map(lambda x: not os.path.exists(x), host_devices)):
if tries >= self.device_scan_attempts:
raise exception.VolumeDeviceNotFound(device=host_devices)
LOG.info(
_LI("ISCSI volume not yet found at: %(host_devices)s. "
"Will rescan & retry. Try number: %(tries)s."), {
'host_devices': host_devices,
'tries': tries
})
# The rescan isn't documented as being necessary(?), but it helps
if self.use_multipath:
self._rescan_iscsi()
# We need to refresh the paths as the devices may be empty
host_devices, target_props = (
self._get_potential_volume_paths(connection_properties))
else:
if (tries):
host_devices = self._get_device_path(target_props)
self._run_iscsiadm(target_props, ("--rescan", ))
tries = tries + 1
if all(map(lambda x: not os.path.exists(x), host_devices)):
time.sleep(tries**2)
else:
break
if tries != 0:
LOG.debug(
"Found iSCSI node %(host_devices)s "
"(after %(tries)s rescans)", {
'host_devices': host_devices,
'tries': tries
})
# Choose an accessible host device
host_device = next(dev for dev in host_devices if os.path.exists(dev))
# find out the WWN of the device
device_wwn = self._linuxscsi.get_scsi_wwn(host_device)
LOG.debug("Device WWN = '%(wwn)s'", {'wwn': device_wwn})
device_info['scsi_wwn'] = device_wwn
if self.use_multipath:
(host_device, multipath_id) = (super(ISCSIConnector,
self)._discover_mpath_device(
device_wwn,
connection_properties,
host_device))
if multipath_id:
device_info['multipath_id'] = multipath_id
device_info['path'] = host_device
LOG.debug("connect_volume returning %s", device_info)
return device_info
def _trace_test_method(*args, **kwargs):
raise exception.VolumeDeviceNotFound('test message')
def fails_once():
self.counter += 1
if self.counter < 2:
raise exception.VolumeDeviceNotFound(device='fake')
else:
return 'success'
def _connect_multipath_volume(self, connection_properties):
"""Connect to a multipathed volume launching parallel login requests.
We will be doing parallel login requests, which will considerably speed
up the process when we have flaky connections.
We'll always try to return a multipath device even if there's only one
path discovered, that way we can return once we have logged in in all
the portals, because the paths will come up later.
To make this possible we tell multipathd that the wwid is a multipath
as soon as we have one device, and then hint multipathd to reconsider
that volume for a multipath asking to add the path, because even if
it's already known by multipathd it would have been discarded if it
was the first time this volume was seen here.
"""
wwn = mpath = None
wwn_added = last_try_on = False
found = []
just_added_devices = []
# Dict used to communicate with threads as detailed in _connect_vol
data = {'stop_connecting': False, 'num_logins': 0, 'failed_logins': 0,
'stopped_threads': 0, 'found_devices': found,
'just_added_devices': just_added_devices}
ips_iqns_luns = self._get_ips_iqns_luns(connection_properties)
# Launch individual threads for each session with the own properties
retries = self.device_scan_attempts
threads = []
for ip, iqn, lun in ips_iqns_luns:
props = connection_properties.copy()
props.update(target_portal=ip, target_iqn=iqn, target_lun=lun)
threads.append(executor.Thread(target=self._connect_vol,
args=(retries, props, data)))
for thread in threads:
thread.start()
# Continue until:
# - All connection attempts have finished and none has logged in
# - Multipath has been found and connection attempts have either
# finished or have already logged in
# - We have finished in all threads, logged in, found some device, and
# 10 seconds have passed, which should be enough with up to 10%
# network package drops.
while not ((len(ips_iqns_luns) == data['stopped_threads'] and
not found) or
(mpath and len(ips_iqns_luns) == data['num_logins'] +
data['failed_logins'])):
# We have devices but we don't know the wwn yet
if not wwn and found:
wwn = self._linuxscsi.get_sysfs_wwn(found)
# We have the wwn but not a multipath
if wwn and not mpath:
mpath = self._linuxscsi.find_sysfs_multipath_dm(found)
if not (mpath or wwn_added):
# Tell multipathd that this wwn is a multipath and hint
# multipathd to recheck all the devices we have just
# connected. We only do this once, since for any new
# device multipathd will already know it is a multipath.
# This is only useful if we have multipathd configured with
# find_multipaths set to yes, and has no effect if it's set
# to no.
wwn_added = self._linuxscsi.multipath_add_wwid(wwn)
while not mpath and just_added_devices:
device_path = '/dev/' + just_added_devices.pop(0)
self._linuxscsi.multipath_add_path(device_path)
mpath = self._linuxscsi.find_sysfs_multipath_dm(found)
# Give some extra time after all threads have finished.
if (not last_try_on and found and
len(ips_iqns_luns) == data['stopped_threads']):
LOG.debug('All connection threads finished, giving 10 seconds '
'for dm to appear.')
last_try_on = time.time() + 10
elif last_try_on and last_try_on < time.time():
break
time.sleep(1)
data['stop_connecting'] = True
for thread in threads:
thread.join()
# If we haven't found any devices let the caller do the cleanup
if not found:
raise exception.VolumeDeviceNotFound(device='')
# NOTE(geguileo): If we cannot find the dm it's because all paths are
# really bad, so we might as well raise a not found exception, but
# in our best effort we'll return a device even if it's probably
# useless.
if not mpath:
LOG.warning('No dm was created, connection to volume is probably '
'bad and will perform poorly.')
return self._get_connect_result(connection_properties, wwn, found,
mpath)
