def _cleanup_caches_if_required(ctx, cache, images_info):
# NOTE(dtantsur): I'd prefer to have this code inside ImageCache. But:
# To reclaim disk space efficiently, this code needs to be aware of
# all existing caches (e.g. cleaning instance image cache can be
# much more efficient, than cleaning TFTP cache).
total_size = sum(images.download_size(ctx, uuid)
for (uuid, path) in images_info)
free = _free_disk_space_for(cache.master_dir)
if total_size >= free:
# NOTE(dtantsur): instance cache is larger - always clean it first
# NOTE(dtantsur): filter caches, whose directory is on the same device
st_dev = os.stat(cache.master_dir).st_dev
caches = [c for c in (InstanceImageCache(), TFTPImageCache())
if os.stat(c.master_dir).st_dev == st_dev]
for cache_to_clean in caches:
# NOTE(dtantsur): multiplying by 2 is an attempt to account for
# images converting to raw format
cache_to_clean.clean_up(amount=(2 * total_size - free))
free = _free_disk_space_for(cache.master_dir)
if total_size < free:
break
else:
msg = _("Disk volume where '%(path)s' is located doesn't have "
"enough disk space. Required %(required)d MiB, "
"only %(actual)d MiB available space present.")
raise exception.InstanceDeployFailure(reason=msg % {
'path': cache.master_dir,
'required': total_size / 1024 / 1024,
'actual': free / 1024 / 1024
})
def _cleanup_caches_if_required(ctx, cache, images_info):
# NOTE(dtantsur): I'd prefer to have this code inside ImageCache. But:
# To reclaim disk space efficiently, this code needs to be aware of
# all existing caches (e.g. cleaning instance image cache can be
# much more efficient, than cleaning TFTP cache).
total_size = sum(
images.download_size(ctx, uuid) for (uuid, path) in images_info)
free = _free_disk_space_for(cache.master_dir)
if total_size >= free:
# NOTE(dtantsur): instance cache is larger - always clean it first
# NOTE(dtantsur): filter caches, whose directory is on the same device
st_dev = os.stat(cache.master_dir).st_dev
caches = [
c for c in (InstanceImageCache(), TFTPImageCache())
if os.stat(c.master_dir).st_dev == st_dev
]
for cache_to_clean in caches:
# NOTE(dtantsur): multiplying by 2 is an attempt to account for
# images converting to raw format
cache_to_clean.clean_up(amount=(2 * total_size - free))
free = _free_disk_space_for(cache.master_dir)
if total_size < free:
break
else:
msg = _("Disk volume where '%(path)s' is located doesn't have "
"enough disk space. Required %(required)d MiB, "
"only %(actual)d MiB available space present.")
raise exception.InstanceDeployFailure(
reason=msg % {
'path': cache.master_dir,
'required': total_size / 1024 / 1024,
'actual': free / 1024 / 1024
})
def check_image_size(task, image_source):
"""Check if the requested image is larger than the ram size.
:param task: a TaskManager instance containing the node to act on.
:param image_source: href of the image.
:raises: InvalidParameterValue if size of the image is greater than
the available ram size.
"""
properties = task.node.properties
# skip check if 'memory_mb' is not defined
if 'memory_mb' not in properties:
LOG.warning(
_LW('Skip the image size check as memory_mb is not '
'defined in properties on node %s.'), task.node.uuid)
return
memory_size = int(properties.get('memory_mb'))
image_size = int(images.download_size(task.context, image_source))
reserved_size = CONF.agent.memory_consumed_by_agent
if (image_size + (reserved_size * units.Mi)) > (memory_size * units.Mi):
msg = (_('Memory size is too small for requested image, if it is '
'less than (image size + reserved RAM size), will break '
'the IPA deployments. Image size: %(image_size)d MiB, '
'Memory size: %(memory_size)d MiB, Reserved size: '
'%(reserved_size)d MiB.') % {
'image_size': image_size / units.Mi,
'memory_size': memory_size,
'reserved_size': reserved_size
})
raise exception.InvalidParameterValue(msg)
def check_image_size(task, image_source):
"""Check if the requested image is larger than the ram size.
:param task: a TaskManager instance containing the node to act on.
:param image_source: href of the image.
:raises: InvalidParameterValue if size of the image is greater than
the available ram size.
"""
properties = task.node.properties
# skip check if 'memory_mb' is not defined
if 'memory_mb' not in properties:
LOG.warning(_LW('Skip the image size check as memory_mb is not '
'defined in properties on node %s.'), task.node.uuid)
return
memory_size = int(properties.get('memory_mb'))
image_size = int(images.download_size(task.context, image_source))
reserved_size = CONF.agent.memory_consumed_by_agent
if (image_size + (reserved_size * units.Mi)) > (memory_size * units.Mi):
msg = (_('Memory size is too small for requested image, if it is '
'less than (image size + reserved RAM size), will break '
'the IPA deployments. Image size: %(image_size)d MiB, '
'Memory size: %(memory_size)d MiB, Reserved size: '
'%(reserved_size)d MiB.')
% {'image_size': image_size / units.Mi,
'memory_size': memory_size,
'reserved_size': reserved_size})
raise exception.InvalidParameterValue(msg)
def clean_up_caches(ctx, directory, images_info):
"""Explicitly cleanup caches based on their priority (if required).
This cleans up the caches to free up the amount of space required for the
images in images_info. The caches are cleaned up one after the other in
the order of their priority. If we still cannot free up enough space
after trying all the caches, this method throws exception.
:param ctx: context
:param directory: the directory (of the cache) to be freed up.
:param images_info: a list of tuples of the form (image_uuid,path)
for which space is to be created in cache.
:raises: InsufficientDiskSpace exception, if we cannot free up enough space
after trying all the caches.
"""
total_size = sum(images.download_size(ctx, uuid) for (uuid, path) in images_info)
_clean_up_caches(directory, total_size)
def clean_up_caches(ctx, directory, images_info):
"""Explicitly cleanup caches based on their priority (if required).
This cleans up the caches to free up the amount of space required for the
images in images_info. The caches are cleaned up one after the other in
the order of their priority. If we still cannot free up enough space
after trying all the caches, this method throws exception.
:param ctx: context
:param directory: the directory (of the cache) to be freed up.
:param images_info: a list of tuples of the form (image_uuid,path)
for which space is to be created in cache.
:raises: InsufficientDiskSpace exception, if we cannot free up enough space
after trying all the caches.
"""
total_size = sum(
images.download_size(ctx, uuid) for (uuid, path) in images_info)
_clean_up_caches(directory, total_size)
def clean_up_caches(ctx, directory, images_info):
"""Explicitly cleanup caches based on their priority (if required).
This cleans up the caches to free up the amount of space required for the
images in images_info. The caches are cleaned up one after the other in
the order of their priority. If we still cannot free up enough space
after trying all the caches, this method throws exception.
:param ctx: context
:param directory: the directory (of the cache) to be freed up.
:param images_info: a list of tuples of the form (image_uuid,path)
for which space is to be created in cache.
:raises: InsufficientDiskSpace exception, if we cannot free up enough space
after trying all the caches.
"""
total_size = sum(images.download_size(ctx, uuid)
for (uuid, path) in images_info)
free = _free_disk_space_for(directory)
if total_size >= free:
# NOTE(dtantsur): filter caches, whose directory is on the same device
st_dev = os.stat(directory).st_dev
caches_to_clean = [x[1]() for x in _cache_cleanup_list]
caches = (c for c in caches_to_clean
if os.stat(c.master_dir).st_dev == st_dev)
for cache_to_clean in caches:
# NOTE(dtantsur): multiplying by 2 is an attempt to account for
# images converting to raw format
cache_to_clean.clean_up(amount=(2 * total_size - free))
free = _free_disk_space_for(directory)
if total_size < free:
break
else:
raise exception.InsufficientDiskSpace(path=directory,
required=total_size / 1024 / 1024,
actual=free / 1024 / 1024,
)
def test_download_size_image_service(self):
image_service_mock = mock.MagicMock()
images.download_size('context', 'image_href', image_service_mock)
image_service_mock.show.assert_called_once_with('image_href')
def test_download_size_no_image_service(self, image_service_mock):
images.download_size('context', 'image_href')
image_service_mock.assert_called_once_with('image_href',
context='context')
image_service_mock.return_value.show.assert_called_once_with(
'image_href')
def test_download_size_image_service(self):
image_service_mock = mock.MagicMock()
images.download_size('context', 'image_href', image_service_mock)
image_service_mock.show.assert_called_once_with('image_href')
def test_download_size_no_image_service(self, image_service_mock):
images.download_size('context', 'image_href')
image_service_mock.assert_called_once_with(version=1,
context='context')
image_service_mock.return_value.show.assert_called_once_with(
'image_href')
def _calculate_memory_req(task):
image_source = task.node.instance_info['image_source']
image_size = images.download_size(task.context, image_source)
return image_size // units.Mi + CONF.ansible.extra_memory
def test_download_size_image_service(self):
image_service_mock = mock.MagicMock()
images.download_size("context", "image_href", image_service_mock)
image_service_mock.show.assert_called_once_with("image_href")
def test_download_size_no_image_service(self, image_service_mock):
images.download_size("context", "image_href")
image_service_mock.assert_called_once_with("image_href", context="context")
image_service_mock.return_value.show.assert_called_once_with("image_href")
def _calculate_memory_req(task):
image_source = task.node.instance_info['image_source']
image_size = images.download_size(task.context, image_source)
return image_size // units.Mi + CONF.ansible.extra_memory
def test_download_size(self, show_mock):
show_mock.return_value = {'size': 123456}
size = images.download_size('context', 'image_href', 'image_service')
self.assertEqual(123456, size)
show_mock.assert_called_once_with('context', 'image_href',
'image_service')
def test_download_size(self, show_mock):
show_mock.return_value = {'size': 123456}
size = images.download_size('context', 'image_href', 'image_service')
self.assertEqual(123456, size)
show_mock.assert_called_once_with('context', 'image_href',
'image_service')