def list_partitions(device):
"""Get partitions information from given device.
:param device: The device path.
:returns: list of dictionaries (one per partition) with keys:
number, start, end, size (in MiB), filesystem, flags
"""
output = utils.execute(
'parted', '-s', '-m', device, 'unit', 'MiB', 'print',
use_standard_locale=True, run_as_root=True)[0]
if isinstance(output, bytes):
output = output.decode("utf-8")
lines = [line for line in output.split('\n') if line.strip()][2:]
# Example of line: 1:1.00MiB:501MiB:500MiB:ext4::boot
fields = ('number', 'start', 'end', 'size', 'filesystem', 'flags')
result = []
for line in lines:
match = _PARTED_PRINT_RE.match(line)
if match is None:
LOG.warning(_LW("Partition information from parted for device "
"%(device)s does not match "
"expected format: %(line)s"),
dict(device=device, line=line))
continue
# Cast int fields to ints (some are floats and we round them down)
groups = [int(float(x)) if i < 4 else x
for i, x in enumerate(match.groups())]
result.append(dict(zip(fields, groups)))
return result
def _send(self, name, value, metric_type, sample_rate=None):
"""Send metrics to the statsd backend
:param name: Metric name
:param value: Metric value
:param metric_type: Metric type (GAUGE_TYPE, COUNTER_TYPE,
or TIMER_TYPE)
:param sample_rate: Probabilistic rate at which the values will be sent
"""
if sample_rate is None:
metric = "%s:%s|%s" % (name, value, metric_type)
else:
metric = "%s:%s|%s@%s" % (name, value, metric_type, sample_rate)
# Ideally, we'd cache a sending socket in self, but that
# results in a socket getting shared by multiple green threads.
with contextlib.closing(self._open_socket()) as sock:
try:
sock.settimeout(0.0)
sock.sendto(metric, self._target)
except socket.error as e:
LOG.warning(
_LW("Failed to send the metric value to " "host %(host)s port %(port)s. " "Error: %(error)s"),
{"host": self._host, "port": self._port, "error": e},
)
def _send(self, name, value, metric_type, sample_rate=None):
"""Send metrics to the statsd backend
:param name: Metric name
:param value: Metric value
:param metric_type: Metric type (GAUGE_TYPE, COUNTER_TYPE,
or TIMER_TYPE)
:param sample_rate: Probabilistic rate at which the values will be sent
"""
if sample_rate is None:
metric = '%s:%s|%s' % (name, value, metric_type)
else:
metric = '%s:%s|%s@%s' % (name, value, metric_type, sample_rate)
# Ideally, we'd cache a sending socket in self, but that
# results in a socket getting shared by multiple green threads.
with contextlib.closing(self._open_socket()) as sock:
try:
sock.settimeout(0.0)
sock.sendto(metric, self._target)
except socket.error as e:
LOG.warning(
_LW("Failed to send the metric value to "
"host %(host)s port %(port)s. "
"Error: %(error)s"), {
'host': self._host,
'port': self._port,
'error': e
})
def _wait_for_disk_to_become_available(self, retries, max_retries, pids,
stderr):
retries[0] += 1
if retries[0] > max_retries:
raise loopingcall.LoopingCallDone()
try:
# NOTE(ifarkas): fuser returns a non-zero return code if none of
# the specified files is accessed
out, err = utils.execute('fuser',
self._device,
check_exit_code=[0, 1],
run_as_root=True)
if not out and not err:
raise loopingcall.LoopingCallDone()
else:
if err:
stderr[0] = err
if out:
pids_match = re.search(self._fuser_pids_re, out)
pids[0] = pids_match.group()
except processutils.ProcessExecutionError as exc:
LOG.warning(
_LW('Failed to check the device %(device)s with fuser:'******' %(err)s'), {
'device': self._device,
'err': exc
})
def unlink_without_raise(path):
try:
os.unlink(path)
except OSError as e:
if e.errno == errno.ENOENT:
return
else:
LOG.warning(_LW("Failed to unlink %(path)s, error: %(e)s"),
{'path': path, 'e': e})
def unlink_without_raise(path):
try:
os.unlink(path)
except OSError as e:
if e.errno == errno.ENOENT:
return
else:
LOG.warning(_LW("Failed to unlink %(path)s, error: %(e)s"),
{'path': path, 'e': e})
def _wait_for_disk_to_become_available(self, retries, max_retries, pids,
stderr):
retries[0] += 1
if retries[0] > max_retries:
raise loopingcall.LoopingCallDone()
try:
# NOTE(ifarkas): fuser returns a non-zero return code if none of
# the specified files is accessed
out, err = utils.execute('fuser', self._device,
check_exit_code=[0, 1], run_as_root=True)
if not out and not err:
raise loopingcall.LoopingCallDone()
else:
if err:
stderr[0] = err
if out:
pids_match = re.search(self._fuser_pids_re, out)
pids[0] = pids_match.group()
except processutils.ProcessExecutionError as exc:
LOG.warning(_LW('Failed to check the device %(device)s with fuser:'******' %(err)s'), {'device': self._device, 'err': exc})
def list_partitions(device):
"""Get partitions information from given device.
:param device: The device path.
:returns: list of dictionaries (one per partition) with keys:
number, start, end, size (in MiB), filesystem, flags
"""
output = utils.execute('parted',
'-s',
'-m',
device,
'unit',
'MiB',
'print',
use_standard_locale=True,
run_as_root=True)[0]
if isinstance(output, bytes):
output = output.decode("utf-8")
lines = [line for line in output.split('\n') if line.strip()][2:]
# Example of line: 1:1.00MiB:501MiB:500MiB:ext4::boot
fields = ('number', 'start', 'end', 'size', 'filesystem', 'flags')
result = []
for line in lines:
match = _PARTED_PRINT_RE.match(line)
if match is None:
LOG.warning(
_LW("Partition information from parted for device "
"%(device)s does not match "
"expected format: %(line)s"), dict(device=device,
line=line))
continue
# Cast int fields to ints (some are floats and we round them down)
groups = [
int(float(x)) if i < 4 else x for i, x in enumerate(match.groups())
]
result.append(dict(zip(fields, groups)))
return result
def match_root_device_hints(devices, root_device_hints):
"""Try to find a device that matches the root device hints.
Try to find a device that matches the root device hints. In order
for a device to be matched it needs to satisfy all the given hints.
:param devices: A list of dictionaries representing the devices
containing one or more of the following keys:
:name: (String) The device name, e.g /dev/sda
:size: (Integer) Size of the device in *bytes*
:model: (String) Device model
:vendor: (String) Device vendor name
:serial: (String) Device serial number
:wwn: (String) Unique storage identifier
:wwn_with_extension: (String): Unique storage identifier with
the vendor extension appended
:wwn_vendor_extension: (String): United vendor storage identifier
:rotational: (Boolean) Whether it's a rotational device or
not. Useful to distinguish HDDs (rotational) and SSDs
(not rotational).
:hctl: (String): The SCSI address: Host, channel, target and lun.
For example: '1:0:0:0'.
:param root_device_hints: A dictionary with the root device hints.
:raises: ValueError, if some information is invalid.
:returns: The first device to match all the hints or None.
"""
LOG.debug('Trying to find a device from "%(devs)s" that matches the '
'root device hints "%(hints)s"', {'devs': devices,
'hints': root_device_hints})
parsed_hints = parse_root_device_hints(root_device_hints)
for dev in devices:
for hint in parsed_hints:
hint_type = VALID_ROOT_DEVICE_HINTS[hint]
device_value = dev.get(hint)
hint_value = parsed_hints[hint]
if hint_type is str:
try:
device_value = _normalize_hint_expression(device_value,
hint)
except ValueError:
LOG.warning(
_LW('The attribute "%(attr)s" of the device "%(dev)s" '
'has an empty value. Skipping device.'),
{'attr': hint, 'dev': dev})
break
# NOTE(lucasagomes): Boolean hints are not supported by
# specs_matcher.match(), so we need to do the comparison
# ourselves
if hint_type is bool:
try:
device_value = strutils.bool_from_string(device_value,
strict=True)
except ValueError:
LOG.warning(_LW('The attribute "%(attr)s" (with value '
'"%(value)s") of device "%(dev)s" is not '
'a valid Boolean. Skipping device.'),
{'attr': hint, 'value': device_value,
'dev': dev})
break
if device_value == hint_value:
continue
break
if hint == 'size':
# Since we don't support units yet we expect the size
# in GiB for now
device_value = device_value / units.Gi
if not specs_matcher.match(device_value, hint_value):
break
else:
LOG.debug('Device found! The device "%s" matches the root '
'device hints' % dev)
return dev
LOG.debug('No device found that matches the root device hints')
def create_config_drive_partition(node_uuid, device, configdrive):
"""Create a partition for config drive
Checks if the device is GPT or MBR partitioned and creates config drive
partition accordingly.
:param node_uuid: UUID of the Node.
:param device: The device path.
:param configdrive: Base64 encoded Gzipped configdrive content or
configdrive HTTP URL.
:raises: InstanceDeployFailure if config drive size exceeds maximum limit
or if it fails to create config drive.
"""
confdrive_file = None
try:
config_drive_part = _get_labelled_partition(device,
CONFIGDRIVE_LABEL,
node_uuid)
confdrive_mb, confdrive_file = _get_configdrive(configdrive,
node_uuid)
if confdrive_mb > MAX_CONFIG_DRIVE_SIZE_MB:
raise exception.InstanceDeployFailure(
_('Config drive size exceeds maximum limit of 64MiB. '
'Size of the given config drive is %(size)d MiB for '
'node %(node)s.')
% {'size': confdrive_mb, 'node': node_uuid})
LOG.debug("Adding config drive partition %(size)d MiB to "
"device: %(dev)s for node %(node)s",
{'dev': device, 'size': confdrive_mb, 'node': node_uuid})
if config_drive_part:
LOG.debug("Configdrive for node %(node)s exists at "
"%(part)s",
{'node': node_uuid, 'part': config_drive_part})
else:
cur_parts = set(part['number'] for part in list_partitions(device))
if _is_disk_gpt_partitioned(device, node_uuid):
_fix_gpt_structs(device, node_uuid)
create_option = '0:-%dMB:0' % MAX_CONFIG_DRIVE_SIZE_MB
utils.execute('sgdisk', '-n', create_option, device,
run_as_root=True)
else:
# Check if the disk has 4 partitions. The MBR based disk
# cannot have more than 4 partitions.
# TODO(stendulker): One can use logical partitions to create
# a config drive if there are 3 primary partitions.
# https://bugs.launchpad.net/ironic/+bug/1561283
try:
pp_count, lp_count = count_mbr_partitions(device)
except ValueError as e:
raise exception.InstanceDeployFailure(
_('Failed to check the number of primary partitions '
'present on %(dev)s for node %(node)s. Error: '
'%(error)s') % {'dev': device, 'node': node_uuid,
'error': e})
if pp_count > 3:
raise exception.InstanceDeployFailure(
_('Config drive cannot be created for node %(node)s. '
'Disk (%(dev)s) uses MBR partitioning and already '
'has %(parts)d primary partitions.')
% {'node': node_uuid, 'dev': device,
'parts': pp_count})
# Check if disk size exceeds 2TB msdos limit
startlimit = '-%dMiB' % MAX_CONFIG_DRIVE_SIZE_MB
endlimit = '-0'
if _is_disk_larger_than_max_size(device, node_uuid):
# Need to create a small partition at 2TB limit
LOG.warning(_LW("Disk size is larger than 2TB for "
"node %(node)s. Creating config drive "
"at the end of the disk %(disk)s."),
{'node': node_uuid, 'disk': device})
startlimit = (MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR -
MAX_CONFIG_DRIVE_SIZE_MB - 1)
endlimit = MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR - 1
utils.execute('parted', '-a', 'optimal', '-s', '--', device,
'mkpart', 'primary', 'ext2', startlimit,
endlimit, run_as_root=True)
upd_parts = set(part['number'] for part in list_partitions(device))
new_part = set(upd_parts) - set(cur_parts)
if len(new_part) != 1:
raise exception.InstanceDeployFailure(
_('Disk partitioning failed on device %(device)s. '
'Unable to retrieve config drive partition information.')
% {'device': device})
if is_iscsi_device(device, node_uuid):
config_drive_part = '%s-part%s' % (device, new_part.pop())
else:
config_drive_part = '%s%s' % (device, new_part.pop())
# NOTE(vdrok): the partition was created successfully, let's wait
# for it to appear in /dev.
LOG.debug('Waiting for the config drive partition %(part)s '
'on node %(node)s to be ready for writing.',
{'part': config_drive_part, 'node': node_uuid})
utils.execute('udevadm', 'settle',
'--exit-if-exists=%s' % config_drive_part)
dd(confdrive_file, config_drive_part)
LOG.info(_LI("Configdrive for node %(node)s successfully "
"copied onto partition %(part)s"),
{'node': node_uuid, 'part': config_drive_part})
except (processutils.UnknownArgumentError,
processutils.ProcessExecutionError, OSError) as e:
msg = (_('Failed to create config drive on disk %(disk)s '
'for node %(node)s. Error: %(error)s') %
{'disk': device, 'node': node_uuid, 'error': e})
LOG.error(msg)
raise exception.InstanceDeployFailure(msg)
finally:
# If the configdrive was requested make sure we delete the file
# after copying the content to the partition
if confdrive_file:
utils.unlink_without_raise(confdrive_file)
def match_root_device_hints(devices, root_device_hints):
"""Try to find a device that matches the root device hints.
Try to find a device that matches the root device hints. In order
for a device to be matched it needs to satisfy all the given hints.
:param devices: A list of dictionaries representing the devices
containing one or more of the following keys:
:name: (String) The device name, e.g /dev/sda
:size: (Integer) Size of the device in *bytes*
:model: (String) Device model
:vendor: (String) Device vendor name
:serial: (String) Device serial number
:wwn: (String) Unique storage identifier
:wwn_with_extension: (String): Unique storage identifier with
the vendor extension appended
:wwn_vendor_extension: (String): United vendor storage identifier
:rotational: (Boolean) Whether it's a rotational device or
not. Useful to distinguish HDDs (rotational) and SSDs
(not rotational).
:param root_device_hints: A dictionary with the root device hints.
:raises: ValueError, if some information is invalid.
:returns: The first device to match all the hints or None.
"""
LOG.debug('Trying to find a device from "%(devs)s" that matches the '
'root device hints "%(hints)s"', {'devs': devices,
'hints': root_device_hints})
parsed_hints = parse_root_device_hints(root_device_hints)
for dev in devices:
for hint in parsed_hints:
hint_type = VALID_ROOT_DEVICE_HINTS[hint]
device_value = dev.get(hint)
hint_value = parsed_hints[hint]
if hint_type is str:
try:
device_value = _normalize_hint_expression(device_value,
hint)
except ValueError:
LOG.warning(
_LW('The attribute "%(attr)s" of the device "%(dev)s" '
'has an empty value. Skipping device.'),
{'attr': hint, 'dev': dev})
break
# NOTE(lucasagomes): Boolean hints are not supported by
# specs_matcher.match(), so we need to do the comparison
# ourselves
if hint_type is bool:
try:
device_value = strutils.bool_from_string(device_value,
strict=True)
except ValueError:
LOG.warning(_LW('The attribute "%(attr)s" (with value '
'"%(value)s") of device "%(dev)s" is not '
'a valid Boolean. Skipping device.'),
{'attr': hint, 'value': device_value,
'dev': dev})
break
if device_value == hint_value:
continue
break
if hint == 'size':
# Since we don't support units yet we expect the size
# in GiB for now
device_value = device_value / units.Gi
if not specs_matcher.match(device_value, hint_value):
break
else:
LOG.debug('Device found! The device "%s" matches the root '
'device hints' % dev)
return dev
LOG.debug('No device found that matches the root device hints')
def create_config_drive_partition(node_uuid, device, configdrive):
"""Create a partition for config drive
Checks if the device is GPT or MBR partitioned and creates config drive
partition accordingly.
:param node_uuid: UUID of the Node.
:param device: The device path.
:param configdrive: Base64 encoded Gzipped configdrive content or
configdrive HTTP URL.
:raises: InstanceDeployFailure if config drive size exceeds maximum limit
or if it fails to create config drive.
"""
confdrive_file = None
try:
config_drive_part = _get_labelled_partition(device, CONFIGDRIVE_LABEL,
node_uuid)
confdrive_mb, confdrive_file = _get_configdrive(configdrive, node_uuid)
if confdrive_mb > MAX_CONFIG_DRIVE_SIZE_MB:
raise exception.InstanceDeployFailure(
_('Config drive size exceeds maximum limit of 64MiB. '
'Size of the given config drive is %(size)d MiB for '
'node %(node)s.') % {
'size': confdrive_mb,
'node': node_uuid
})
LOG.debug(
"Adding config drive partition %(size)d MiB to "
"device: %(dev)s for node %(node)s", {
'dev': device,
'size': confdrive_mb,
'node': node_uuid
})
if config_drive_part:
LOG.debug("Configdrive for node %(node)s exists at "
"%(part)s", {
'node': node_uuid,
'part': config_drive_part
})
else:
cur_parts = set(part['number'] for part in list_partitions(device))
if _is_disk_gpt_partitioned(device, node_uuid):
_fix_gpt_structs(device, node_uuid)
create_option = '0:-%dMB:0' % MAX_CONFIG_DRIVE_SIZE_MB
utils.execute('sgdisk',
'-n',
create_option,
device,
run_as_root=True)
else:
# Check if the disk has 4 partitions. The MBR based disk
# cannot have more than 4 partitions.
# TODO(stendulker): One can use logical partitions to create
# a config drive if there are 3 primary partitions.
# https://bugs.launchpad.net/ironic/+bug/1561283
try:
pp_count, lp_count = count_mbr_partitions(device)
except ValueError as e:
raise exception.InstanceDeployFailure(
_('Failed to check the number of primary partitions '
'present on %(dev)s for node %(node)s. Error: '
'%(error)s') % {
'dev': device,
'node': node_uuid,
'error': e
})
if pp_count > 3:
raise exception.InstanceDeployFailure(
_('Config drive cannot be created for node %(node)s. '
'Disk (%(dev)s) uses MBR partitioning and already '
'has %(parts)d primary partitions.') % {
'node': node_uuid,
'dev': device,
'parts': pp_count
})
# Check if disk size exceeds 2TB msdos limit
startlimit = '-%dMiB' % MAX_CONFIG_DRIVE_SIZE_MB
endlimit = '-0'
if _is_disk_larger_than_max_size(device, node_uuid):
# Need to create a small partition at 2TB limit
LOG.warning(
_LW("Disk size is larger than 2TB for "
"node %(node)s. Creating config drive "
"at the end of the disk %(disk)s."), {
'node': node_uuid,
'disk': device
})
startlimit = (MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR -
MAX_CONFIG_DRIVE_SIZE_MB - 1)
endlimit = MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR - 1
utils.execute('parted',
'-a',
'optimal',
'-s',
'--',
device,
'mkpart',
'primary',
'fat32',
startlimit,
endlimit,
run_as_root=True)
upd_parts = set(part['number'] for part in list_partitions(device))
new_part = set(upd_parts) - set(cur_parts)
if len(new_part) != 1:
raise exception.InstanceDeployFailure(
_('Disk partitioning failed on device %(device)s. '
'Unable to retrieve config drive partition information.')
% {'device': device})
if is_iscsi_device(device, node_uuid):
config_drive_part = '%s-part%s' % (device, new_part.pop())
else:
config_drive_part = '%s%s' % (device, new_part.pop())
# NOTE(vdrok): the partition was created successfully, let's wait
# for it to appear in /dev.
LOG.debug(
'Waiting for the config drive partition %(part)s '
'on node %(node)s to be ready for writing.', {
'part': config_drive_part,
'node': node_uuid
})
utils.execute('udevadm', 'settle',
'--exit-if-exists=%s' % config_drive_part)
dd(confdrive_file, config_drive_part)
LOG.info(
_LI("Configdrive for node %(node)s successfully "
"copied onto partition %(part)s"), {
'node': node_uuid,
'part': config_drive_part
})
except (processutils.UnknownArgumentError,
processutils.ProcessExecutionError, OSError) as e:
msg = (_('Failed to create config drive on disk %(disk)s '
'for node %(node)s. Error: %(error)s') % {
'disk': device,
'node': node_uuid,
'error': e
})
LOG.error(msg)
raise exception.InstanceDeployFailure(msg)
finally:
# If the configdrive was requested make sure we delete the file
# after copying the content to the partition
if confdrive_file:
utils.unlink_without_raise(confdrive_file)