def test_attach_volumes_failure(self, mock_log, mock_attach, mock_detach):
"""Verify detach is called upon attachment failing."""
object_utils.create_test_volume_target(self.context,
node_id=self.node.id,
volume_type='iscsi',
boot_index=0,
volume_id='1234')
object_utils.create_test_volume_target(self.context,
node_id=self.node.id,
volume_type='iscsi',
boot_index=1,
volume_id='5678',
uuid=uuidutils.generate_uuid())
object_utils.create_test_volume_connector(self.context,
node_id=self.node.id,
type='iqn',
connector_id='iqn.address')
mock_attach.side_effect = exception.StorageError('foo')
with task_manager.acquire(self.context, self.node.id) as task:
self.assertRaises(exception.StorageError,
self.interface.attach_volumes, task)
self.assertTrue(mock_attach.called)
self.assertTrue(mock_detach.called)
# Replacing the mock to not return an error, should still raise an
# exception.
mock_attach.reset_mock()
mock_detach.reset_mock()
def test_detach_volumes_failure(self, mock_log, mock_detach):
object_utils.create_test_volume_target(self.context,
node_id=self.node.id,
volume_type='iscsi',
boot_index=0,
volume_id='1234')
object_utils.create_test_volume_connector(self.context,
node_id=self.node.id,
type='iqn',
connector_id='iqn.address')
with task_manager.acquire(self.context, self.node.id) as task:
# The first attempt should succeed.
# The second attempt should throw StorageError
# Third attempt, should log errors but not raise an exception.
mock_detach.side_effect = [
None, exception.StorageError('bar'), None
]
# This should generate 1 mock_detach call and succeed
self.interface.detach_volumes(task)
task.node.provision_state = states.DELETED
# This should generate the other 2 moc_detach calls and warn
self.interface.detach_volumes(task)
self.assertEqual(3, mock_detach.call_count)
self.assertEqual(1, mock_log.warning.call_count)
def populate_storage_driver_internal_info(task):
"""Set node driver_internal_info for boot from volume parameters.
:param task: a TaskManager object containing the node.
:raises: StorageError when a node has an iSCSI or FibreChannel boot volume
defined but is not capable to support it.
"""
node = task.node
boot_volume = get_remote_boot_volume(task)
if not boot_volume:
return
vol_type = str(boot_volume.volume_type).lower()
node_caps = driver_utils.capabilities_to_dict(
node.properties.get('capabilities'))
if vol_type == 'iscsi' and 'iscsi_boot' not in node_caps:
# TODO(TheJulia): In order to support the FCoE and HBA boot cases,
# some additional logic will be needed here to ensure we align.
# The deployment, in theory, should never reach this point
# if the interfaces all validated, but we shouldn't use that
# as the only guard against bad configurations.
raise exception.StorageError(
_('Node %(node)s has an iSCSI boot '
'volume defined and no iSCSI boot '
'support available.') % {'node': node.uuid})
if vol_type == 'fibre_channel' and 'fibre_channel_boot' not in node_caps:
raise exception.StorageError(
_('Node %(node)s has a Fibre Channel '
'boot volume defined and no Fibre '
'Channel boot support available.') % {'node': node.uuid})
boot_capability = ("%s_volume_boot" % vol_type)
deploy_capability = ("%s_volume_deploy" % vol_type)
vol_uuid = boot_volume['uuid']
driver_internal_info = node.driver_internal_info
if check_interface_capability(task.driver.boot, boot_capability):
driver_internal_info['boot_from_volume'] = vol_uuid
# NOTE(TheJulia): This would be a convenient place to check
# if we need to know about deploying the volume.
if (check_interface_capability(task.driver.deploy, deploy_capability)
and task.driver.storage.should_write_image(task)):
driver_internal_info['boot_from_volume_deploy'] = vol_uuid
# NOTE(TheJulia): This is also a useful place to include a
# root device hint since we should/might/be able to obtain
# and supply that information to IPA if it needs to write
# the image to the volume.
node.driver_internal_info = driver_internal_info
node.save()
def attach_volumes(self, task):
"""Informs the storage subsystem to attach all volumes for the node.
:param task: The task object.
:raises: StorageError If an underlying exception or failure
is detected.
"""
node = task.node
targets = [target.volume_id for target in task.volume_targets]
# If there are no targets, then we have nothing to do.
if not targets:
return
connector = self._generate_connector(task)
try:
connected = cinder.attach_volumes(task, targets, connector)
except exception.StorageError as e:
with excutils.save_and_reraise_exception():
LOG.error(
"Error attaching volumes for node %(node)s: "
"%(err)s", {
'node': node.uuid,
'err': e
})
self.detach_volumes(task,
connector=connector,
aborting_attach=True)
if len(targets) != len(connected):
LOG.error(
"The number of volumes defined for node %(node)s does "
"not match the number of attached volumes. Attempting "
"detach and abort operation.", {'node': node.uuid})
self.detach_volumes(task,
connector=connector,
aborting_attach=True)
raise exception.StorageError(
("Mismatch between the number of "
"configured volume targets for "
"node %(uuid)s and the number of "
"completed attachments.") % {'uuid': node.uuid})
for volume in connected:
# Volumes that were already attached are
# skipped. Updating target volume properties
# for these volumes is nova's responsibility.
if not volume.get('already_attached'):
volume_uuid = volume['data']['ironic_volume_uuid']
targets = objects.VolumeTarget.list_by_volume_id(
task.context, volume_uuid)
for target in targets:
target.properties = volume['data']
target.save()
def _init_client(task):
"""Obtain cinder client and return it for use.
:param task: TaskManager instance representing the operation.
:returns: A cinder client.
:raises: StorageError If an exception is encountered creating the client.
"""
node = task.node
try:
return get_client()
except Exception as e:
msg = (_('Failed to initialize cinder client for operations on node '
'%(uuid)s: %(err)s') % {'uuid': node.uuid, 'err': e})
LOG.error(msg)
raise exception.StorageError(msg)
def test_detach_volumes_failure_raises_exception(self,
mock_log,
mock_detach):
object_utils.create_test_volume_target(
self.context, node_id=self.node.id, volume_type='iscsi',
boot_index=0, volume_id='1234')
object_utils.create_test_volume_connector(
self.context, node_id=self.node.id, type='iqn',
connector_id='iqn.address')
with task_manager.acquire(self.context, self.node.id) as task:
mock_detach.side_effect = exception.StorageError('bar')
self.assertRaises(exception.StorageError,
self.interface.detach_volumes,
task)
# Check that we warn every retry except the last one.
self.assertEqual(3, mock_log.warning.call_count)
self.assertEqual(1, mock_log.error.call_count)
# CONF.cinder.action_retries + 1, number of retries is set to 3.
self.assertEqual(4, mock_detach.call_count)
def _handle_errors(msg):
if allow_errors:
LOG.warning(msg)
else:
LOG.error(msg)
raise exception.StorageError(msg)
def attach_volumes(task, volume_list, connector):
"""Attach volumes to a node.
Enumerate through the provided list of volumes and attach the volumes
to the node defined in the task utilizing the provided connector
information.
If an attachment appears to already exist, we will skip attempting to
attach the volume. If use of the volume fails, a user may need to
remove any lingering pre-existing/unused attachment records since
we have no way to validate if the connector profile data differs
from what was provided to cinder.
:param task: TaskManager instance representing the operation.
:param volume_list: List of volume_id UUID values representing volumes.
:param connector: Dictionary object representing the node sufficiently
to attach a volume. This value can vary based upon
the node's configuration, capability, and ultimately
the back-end storage driver. As cinder was designed
around iSCSI, the 'ip' and 'initiator' keys are
generally expected by cinder drivers.
For FiberChannel, the key 'wwpns' can be used
with a list of port addresses.
Some drivers support a 'multipath' boolean key,
although it is generally False. The 'host' key
is generally used for logging by drivers.
Example:
{
'wwpns': ['list','of','port','wwns'],
'ip': 'ip address',
'initiator': 'initiator iqn',
'multipath': False,
'host': 'hostname',
}
:raises: StorageError If storage subsystem exception is raised.
:returns: List of connected volumes, including volumes that were
already connected to desired nodes. The returned list
can be relatively consistent depending on the end storage
driver that the volume is configured for, however
the 'driver_volume_type' key should not be relied upon
as it is a free-form value returned by the driver.
The accompanying 'data' key contains the actual target
details which will indicate either target WWNs and a LUN
or a target portal and IQN. It also always contains
volume ID in cinder and ironic. Except for these two IDs,
each driver may return somewhat different data although
the same keys are used if the target is FC or iSCSI,
so any logic should be based upon the returned contents.
For already attached volumes, the structure contains
'already_attached': True key-value pair. In such case,
connection info for the node is already in the database,
'data' structure contains only basic info of volume ID in
cinder and ironic, so any logic based on that should
retrieve it from the database.
Example:
[{
'driver_volume_type': 'fibre_channel'
'data': {
'encrypted': False,
'target_lun': 1,
'target_wwn': ['1234567890123', '1234567890124'],
'volume_id': '00000000-0000-0000-0000-000000000001',
'ironic_volume_id':
'11111111-0000-0000-0000-000000000001'}
},
{
'driver_volume_type': 'iscsi'
'data': {
'target_iqn': 'iqn.2010-10.org.openstack:volume-00000002',
'target_portal': '127.0.0.0.1:3260',
'volume_id': '00000000-0000-0000-0000-000000000002',
'ironic_volume_id':
'11111111-0000-0000-0000-000000000002',
'target_lun': 2}
},
{
'already_attached': True
'data': {
'volume_id': '00000000-0000-0000-0000-000000000002',
'ironic_volume_id':
'11111111-0000-0000-0000-000000000002'}
}]
"""
node = task.node
client = _init_client(task)
connected = []
for volume_id in volume_list:
try:
volume = client.volumes.get(volume_id)
except cinder_exceptions.ClientException as e:
msg = (_('Failed to get volume %(vol_id)s from cinder for node '
'%(uuid)s: %(err)s') %
{'vol_id': volume_id, 'uuid': node.uuid, 'err': e})
LOG.error(msg)
raise exception.StorageError(msg)
if is_volume_attached(node, volume):
LOG.debug('Volume %(vol_id)s is already attached to node '
'%(uuid)s. Skipping attachment.',
{'vol_id': volume_id, 'uuid': node.uuid})
# NOTE(jtaryma): Actual connection info of already connected
# volume will be provided by nova. Adding this dictionary to
# 'connected' list so it contains also already connected volumes.
connection = {'data': {'ironic_volume_uuid': volume.id,
'volume_id': volume_id},
'already_attached': True}
connected.append(connection)
continue
try:
client.volumes.reserve(volume_id)
except cinder_exceptions.ClientException as e:
msg = (_('Failed to reserve volume %(vol_id)s for node %(node)s: '
'%(err)s)') %
{'vol_id': volume_id, 'node': node.uuid, 'err': e})
LOG.error(msg)
raise exception.StorageError(msg)
try:
# Provide connector information to cinder
connection = client.volumes.initialize_connection(volume_id,
connector)
except cinder_exceptions.ClientException as e:
msg = (_('Failed to initialize connection for volume '
'%(vol_id)s to node %(node)s: %(err)s') %
{'vol_id': volume_id, 'node': node.uuid, 'err': e})
LOG.error(msg)
raise exception.StorageError(msg)
if 'volume_id' not in connection['data']:
connection['data']['volume_id'] = volume_id
connection['data']['ironic_volume_uuid'] = volume.id
connected.append(connection)
LOG.info('Successfully initialized volume %(vol_id)s for '
'node %(node)s.', {'vol_id': volume_id, 'node': node.uuid})
instance_uuid = node.instance_uuid or node.uuid
try:
# NOTE(TheJulia): The final step of the cinder volume
# attachment process involves updating the volume
# database record to indicate that the attachment has
# been completed, which moves the volume to the
# 'attached' state. This action also sets a mountpoint
# for the volume, if known. In our use case, there is
# no way for us to know what the mountpoint is inside of
# the operating system, thus we send None.
client.volumes.attach(volume_id, instance_uuid, None)
except cinder_exceptions.ClientException as e:
msg = (_('Failed to inform cinder that the attachment for volume '
'%(vol_id)s for node %(node)s has been completed: '
'%(err)s') %
{'vol_id': volume_id, 'node': node.uuid, 'err': e})
LOG.error(msg)
raise exception.StorageError(msg)
try:
# Set metadata to assist a user in volume identification
client.volumes.set_metadata(
volume_id,
_create_metadata_dictionary(node, 'attached'))
except cinder_exceptions.ClientException as e:
LOG.warning('Failed to update volume metadata for volume '
'%(vol_id)s for node %(node)s: %(err)s',
{'vol_id': volume_id, 'node': node.uuid, 'err': e})
return connected
def _generate_connector(self, task):
"""Generate cinder connector value based upon the node.
Generates cinder compatible connector information for the purpose of
attaching volumes. Translation: We need to tell the storage where and
possibly how we can connect.
Supports passing iSCSI information in the form of IP and IQN records,
as well as Fibre Channel information in the form of WWPN addresses.
Fibre Channel WWNN addresses are also sent, however at present in-tree
Cinder drivers do not utilize WWNN addresses.
If multiple connectors exist, the request will be filed with
MultiPath IO being enabled.
A warning is logged if an unsupported volume type is encountered.
:params task: The task object.
:returns: A dictionary data structure similar to:
{'ip': ip,
'initiator': iqn,
'multipath: True,
'wwpns': ['WWN1', 'WWN2']}
:raises: StorageError upon no valid connector record being identified.
"""
data = {}
valid = False
for connector in task.volume_connectors:
if 'iqn' in connector.type and 'initiator' not in data:
data['initiator'] = connector.connector_id
valid = True
elif 'ip' in connector.type and 'ip' not in data:
data['ip'] = connector.connector_id
# TODO(TheJulia): Translate to, or generate an IQN.
elif 'wwpn' in connector.type:
data.setdefault('wwpns', []).append(connector.connector_id)
valid = True
elif 'wwnn' in connector.type:
data.setdefault('wwnns', []).append(connector.connector_id)
valid = True
else:
# TODO(jtaryma): Add handling of type 'mac' with MAC to IP
# translation.
LOG.warning(
'Node %(node)s has a volume_connector (%(uuid)s) '
'defined with an unsupported type: %(type)s.', {
'node': task.node.uuid,
'uuid': connector.uuid,
'type': connector.type
})
if not valid:
valid_types = ', '.join(VALID_FC_TYPES + VALID_ISCSI_TYPES)
msg = (_('Insufficient or incompatible volume connection '
'records for node %(uuid)s. Valid connector '
'types: %(types)s') % {
'uuid': task.node.uuid,
'types': valid_types
})
LOG.error(msg)
raise exception.StorageError(msg)
# NOTE(TheJulia): Hostname appears to only be used for logging
# in cinder drivers, however that may not always be true, and
# may need to change over time.
data['host'] = task.node.uuid
if len(task.volume_connectors) > 1 and len(data) > 1:
data['multipath'] = True
return data
