def test__cache_instance_images_master_path(self):
temp_dir = tempfile.mkdtemp()
CONF.set_default('images_path', temp_dir, group='pxe')
CONF.set_default('instance_master_path',
os.path.join(temp_dir, 'instance_master_path'),
group='pxe')
fileutils.ensure_tree(CONF.pxe.instance_master_path)
fd, tmp_master_image = tempfile.mkstemp(
dir=CONF.pxe.instance_master_path)
self.mox.StubOutWithMock(images, 'fetch_to_raw')
self.mox.StubOutWithMock(tempfile, 'mkstemp')
self.mox.StubOutWithMock(service_utils, 'parse_image_ref')
tempfile.mkstemp(dir=CONF.pxe.instance_master_path).\
AndReturn((fd, tmp_master_image))
images.fetch_to_raw(None, 'glance://image_uuid',
tmp_master_image,
None).\
AndReturn(None)
service_utils.parse_image_ref('glance://image_uuid').\
AndReturn(('image_uuid', None, None, None))
self.mox.ReplayAll()
(uuid, image_path) = pxe._cache_instance_image(None, self.node)
self.mox.VerifyAll()
self.assertEqual(uuid, 'glance://image_uuid')
self.assertEqual(image_path, temp_dir + '/fake_instance_name/disk')
def fetch_image(self, uuid, dest_path, ctx=None):
"""Fetch image with given uuid to the destination path.
Does nothing if destination path exists.
Only creates a link if master image for this UUID is already in cache.
Otherwise downloads an image and also stores it in cache.
:param uuid: image UUID or href to fetch
:param dest_path: destination file path
:param ctx: context
"""
img_download_lock_name = 'download-image'
if self.master_dir is None:
#NOTE(ghe): We don't share images between instances/hosts
if not CONF.parallel_image_downloads:
with lockutils.lock(img_download_lock_name, 'ironic-'):
images.fetch_to_raw(ctx, uuid, dest_path,
self._image_service)
else:
images.fetch_to_raw(ctx, uuid, dest_path, self._image_service)
return
#TODO(ghe): have hard links and counts the same behaviour in all fs
master_file_name = service_utils.parse_image_ref(uuid)[0]
master_path = os.path.join(self.master_dir, master_file_name)
if CONF.parallel_image_downloads:
img_download_lock_name = 'download-image:%s' % master_file_name
# TODO(dtantsur): lock expiration time
with lockutils.lock(img_download_lock_name, 'ironic-'):
if os.path.exists(dest_path):
LOG.debug("Destination %(dest)s already exists for "
"image %(uuid)s" % {
'uuid': uuid,
'dest': dest_path
})
return
try:
# NOTE(dtantsur): ensure we're not in the middle of clean up
with lockutils.lock('master_image', 'ironic-'):
os.link(master_path, dest_path)
except OSError:
LOG.info(
_("Master cache miss for image %(uuid)s, "
"starting download") % {'uuid': uuid})
else:
LOG.debug("Master cache hit for image %(uuid)s",
{'uuid': uuid})
return
self._download_image(uuid, master_path, dest_path, ctx=ctx)
# NOTE(dtantsur): we increased cache size - time to clean up
self.clean_up()
def fetch_image(self, uuid, dest_path, ctx=None):
"""Fetch image with given uuid to the destination path.
Does nothing if destination path exists.
Only creates a link if master image for this UUID is already in cache.
Otherwise downloads an image and also stores it in cache.
:param uuid: image UUID or href to fetch
:param dest_path: destination file path
:param ctx: context
"""
img_download_lock_name = 'download-image'
if self.master_dir is None:
#NOTE(ghe): We don't share images between instances/hosts
if not CONF.parallel_image_downloads:
with lockutils.lock(img_download_lock_name, 'ironic-'):
images.fetch_to_raw(ctx, uuid, dest_path,
self._image_service)
else:
images.fetch_to_raw(ctx, uuid, dest_path,
self._image_service)
return
#TODO(ghe): have hard links and counts the same behaviour in all fs
master_file_name = service_utils.parse_image_ref(uuid)[0]
master_path = os.path.join(self.master_dir, master_file_name)
if CONF.parallel_image_downloads:
img_download_lock_name = 'download-image:%s' % master_file_name
# TODO(dtantsur): lock expiration time
with lockutils.lock(img_download_lock_name, 'ironic-'):
if os.path.exists(dest_path):
LOG.debug("Destination %(dest)s already exists for "
"image %(uuid)s" %
{'uuid': uuid,
'dest': dest_path})
return
try:
# NOTE(dtantsur): ensure we're not in the middle of clean up
with lockutils.lock('master_image', 'ironic-'):
os.link(master_path, dest_path)
except OSError:
LOG.info(_("Master cache miss for image %(uuid)s, "
"starting download") %
{'uuid': uuid})
else:
LOG.debug("Master cache hit for image %(uuid)s",
{'uuid': uuid})
return
self._download_image(uuid, master_path, dest_path, ctx=ctx)
# NOTE(dtantsur): we increased cache size - time to clean up
self.clean_up()
def test__cache_tftp_images_no_master_path(self):
temp_dir = tempfile.mkdtemp()
CONF.set_default('tftp_root', temp_dir, group='pxe')
CONF.set_default('tftp_master_path', None, group='pxe')
image_info = {'deploy_kernel': ['deploy_kernel',
os.path.join(temp_dir,
'instance_uuid_123/deploy_kernel')]}
self.mox.StubOutWithMock(images, 'fetch_to_raw')
images.fetch_to_raw(None, 'deploy_kernel',
os.path.join(temp_dir,
'instance_uuid_123/deploy_kernel'),
None).AndReturn(None)
self.mox.ReplayAll()
pxe._cache_tftp_images(None, self.node, image_info)
self.mox.VerifyAll()
def test__cache_instance_images_no_master_path(self):
temp_dir = tempfile.mkdtemp()
CONF.set_default('images_path', temp_dir, group='pxe')
CONF.set_default('instance_master_path', None, group='pxe')
self.mox.StubOutWithMock(images, 'fetch_to_raw')
images.fetch_to_raw(None, 'glance://image_uuid',
os.path.join(temp_dir,
'fake_instance_name/disk'),
None).AndReturn(None)
self.mox.ReplayAll()
(uuid, image_path) = pxe._cache_instance_image(None, self.node)
self.mox.VerifyAll()
self.assertEqual(uuid, 'glance://image_uuid')
self.assertEqual(image_path,
os.path.join(temp_dir,
'fake_instance_name/disk'))
def test__cache_tftp_images_master_path(self):
temp_dir = tempfile.mkdtemp()
CONF.set_default('tftp_root', temp_dir, group='pxe')
CONF.set_default('tftp_master_path', os.path.join(temp_dir,
'tftp_master_path'),
group='pxe')
image_info = {'deploy_kernel': ['deploy_kernel', temp_dir +
'/instance_uuid_123/deploy_kernel']}
fileutils.ensure_tree(CONF.pxe.tftp_master_path)
fd, tmp_master_image = tempfile.mkstemp(dir=CONF.pxe.tftp_master_path)
self.mox.StubOutWithMock(images, 'fetch_to_raw')
self.mox.StubOutWithMock(tempfile, 'mkstemp')
tempfile.mkstemp(dir=CONF.pxe.tftp_master_path).\
AndReturn((fd, tmp_master_image))
images.fetch_to_raw(None, 'deploy_kernel', tmp_master_image, None).\
AndReturn(None)
self.mox.ReplayAll()
pxe._cache_tftp_images(None, self.node, image_info)
self.mox.VerifyAll()
def _download_image(self, uuid, master_path, dest_path, ctx=None):
"""Download image from Glance and store at a given path.
This method should be called with uuid-specific lock taken.
:param uuid: image UUID or href to fetch
:param master_path: destination master path
:param dest_path: destination file path
:param ctx: context
"""
# TODO(ghe): timeout and retry for downloads
# TODO(ghe): logging when image cannot be created
fd, tmp_path = tempfile.mkstemp(dir=self.master_dir)
os.close(fd)
images.fetch_to_raw(ctx, uuid, tmp_path, self._image_service)
# NOTE(dtantsur): no need for global lock here - master_path
# will have link count >1 at any moment, so won't be cleaned up
os.link(tmp_path, master_path)
os.link(master_path, dest_path)
os.unlink(tmp_path)
def _download_image(self, uuid, master_path, dest_path, ctx=None):
"""Download image from Glance and store at a given path.
This method should be called with uuid-specific lock taken.
:param uuid: image UUID or href to fetch
:param master_path: destination master path
:param dest_path: destination file path
:param ctx: context
"""
#TODO(ghe): timeout and retry for downloads
#TODO(ghe): logging when image cannot be created
fd, tmp_path = tempfile.mkstemp(dir=self.master_dir)
os.close(fd)
images.fetch_to_raw(ctx, uuid, tmp_path, self._image_service)
# NOTE(dtantsur): no need for global lock here - master_path
# will have link count >1 at any moment, so won't be cleaned up
os.link(tmp_path, master_path)
os.link(master_path, dest_path)
os.unlink(tmp_path)
def _get_image(ctx, path, uuid, master_path=None, image_service=None):
#TODO(ghe): Revise this logic and cdocument process Bug #1199665
# When master_path defined, we save the images in this dir using the iamge
# uuid as the file name. Deployments that use this images, creates a hard
# link to keep track of this. When the link count of a master image is
# equal to 1, can be deleted.
#TODO(ghe): have hard links and count links the same behaviour in all fs
#TODO(ghe): timeout and retry for downloads
def _wait_for_download():
if not os.path.exists(lock_file):
raise loopingcall.LoopingCallDone()
# If the download of the image needed is in progress (lock file present)
# we wait until the locks disappears and create the link.
if master_path is None:
#NOTE(ghe): We don't share images between instances/hosts
images.fetch_to_raw(ctx, uuid, path, image_service)
else:
master_uuid = os.path.join(master_path,
service_utils.parse_image_ref(uuid)[0])
lock_file = os.path.join(master_path, master_uuid + '.lock')
_link_master_image(master_uuid, path)
if not os.path.exists(path):
fileutils.ensure_tree(master_path)
if not _download_in_progress(lock_file):
with fileutils.remove_path_on_error(lock_file):
#TODO(ghe): logging when image cannot be created
fd, tmp_path = tempfile.mkstemp(dir=master_path)
os.close(fd)
images.fetch_to_raw(ctx, uuid, tmp_path, image_service)
_create_master_image(tmp_path, master_uuid, path)
_remove_download_in_progress_lock(lock_file)
else:
#TODO(ghe): expiration time
timer = loopingcall.FixedIntervalLoopingCall(
_wait_for_download)
timer.start(interval=1).wait()
_link_master_image(master_uuid, path)
def _get_image(ctx, path, uuid, master_path=None, image_service=None):
#TODO(ghe): Revise this logic and cdocument process Bug #1199665
# When master_path defined, we save the images in this dir using the iamge
# uuid as the file name. Deployments that use this images, creates a hard
# link to keep track of this. When the link count of a master image is
# equal to 1, can be deleted.
#TODO(ghe): have hard links and count links the same behaviour in all fs
#TODO(ghe): timeout and retry for downloads
def _wait_for_download():
if not os.path.exists(lock_file):
raise loopingcall.LoopingCallDone()
# If the download of the image needed is in progress (lock file present)
# we wait until the locks disappears and create the link.
if master_path is None:
#NOTE(ghe): We don't share images between instances/hosts
images.fetch_to_raw(ctx, uuid, path, image_service)
else:
master_uuid = os.path.join(master_path,
service_utils.parse_image_ref(uuid)[0])
lock_file = os.path.join(master_path, master_uuid + '.lock')
_link_master_image(master_uuid, path)
if not os.path.exists(path):
fileutils.ensure_tree(master_path)
if not _download_in_progress(lock_file):
with fileutils.remove_path_on_error(lock_file):
#TODO(ghe): logging when image cannot be created
fd, tmp_path = tempfile.mkstemp(dir=master_path)
os.close(fd)
images.fetch_to_raw(ctx, uuid, tmp_path, image_service)
_create_master_image(tmp_path, master_uuid, path)
_remove_download_in_progress_lock(lock_file)
else:
#TODO(ghe): expiration time
timer = loopingcall.FixedIntervalLoopingCall(
_wait_for_download)
timer.start(interval=1).wait()
_link_master_image(master_uuid, path)
def test_fetch_raw_image(self):
def fake_execute(*cmd, **kwargs):
self.executes.append(cmd)
return None, None
def fake_rename(old, new):
self.executes.append(('mv', old, new))
def fake_unlink(path):
self.executes.append(('rm', path))
@contextlib.contextmanager
def fake_rm_on_error(path):
try:
yield
except Exception:
with excutils.save_and_reraise_exception():
fake_del_if_exists(path)
def fake_del_if_exists(path):
self.executes.append(('rm', '-f', path))
def fake_qemu_img_info(path):
class FakeImgInfo(object):
pass
file_format = path.split('.')[-1]
if file_format == 'part':
file_format = path.split('.')[-2]
elif file_format == 'converted':
file_format = 'raw'
if 'backing' in path:
backing_file = 'backing'
else:
backing_file = None
FakeImgInfo.file_format = file_format
FakeImgInfo.backing_file = backing_file
return FakeImgInfo()
self.useFixture(fixtures.MonkeyPatch(
'ironic.common.utils.execute', fake_execute))
self.useFixture(fixtures.MonkeyPatch('os.rename', fake_rename))
self.useFixture(fixtures.MonkeyPatch('os.unlink', fake_unlink))
self.useFixture(fixtures.MonkeyPatch(
'ironic.common.images.fetch', lambda *_: None))
self.useFixture(fixtures.MonkeyPatch(
'ironic.common.images.qemu_img_info', fake_qemu_img_info))
self.useFixture(fixtures.MonkeyPatch(
'ironic.openstack.common.fileutils.remove_path_on_error',
fake_rm_on_error))
self.useFixture(fixtures.MonkeyPatch(
'ironic.openstack.common.fileutils.delete_if_exists',
fake_del_if_exists))
context = 'opaque context'
image_id = '4'
target = 't.qcow2'
self.executes = []
expected_commands = [('qemu-img', 'convert', '-O', 'raw',
't.qcow2.part', 't.qcow2.converted'),
('rm', 't.qcow2.part'),
('mv', 't.qcow2.converted', 't.qcow2')]
images.fetch_to_raw(context, image_id, target)
self.assertEqual(expected_commands, self.executes)
target = 't.raw'
self.executes = []
expected_commands = [('mv', 't.raw.part', 't.raw')]
images.fetch_to_raw(context, image_id, target)
self.assertEqual(expected_commands, self.executes)
target = 'backing.qcow2'
self.executes = []
expected_commands = [('rm', '-f', 'backing.qcow2.part')]
self.assertRaises(exception.ImageUnacceptable,
images.fetch_to_raw,
context, image_id, target)
self.assertEqual(expected_commands, self.executes)
del self.executes
def test_fetch_raw_image(self):
def fake_execute(*cmd, **kwargs):
self.executes.append(cmd)
return None, None
def fake_rename(old, new):
self.executes.append(('mv', old, new))
def fake_unlink(path):
self.executes.append(('rm', path))
@contextlib.contextmanager
def fake_rm_on_error(path):
try:
yield
except Exception:
with excutils.save_and_reraise_exception():
fake_del_if_exists(path)
def fake_del_if_exists(path):
self.executes.append(('rm', '-f', path))
def fake_qemu_img_info(path):
class FakeImgInfo(object):
pass
file_format = path.split('.')[-1]
if file_format == 'part':
file_format = path.split('.')[-2]
elif file_format == 'converted':
file_format = 'raw'
if 'backing' in path:
backing_file = 'backing'
else:
backing_file = None
FakeImgInfo.file_format = file_format
FakeImgInfo.backing_file = backing_file
return FakeImgInfo()
self.useFixture(
fixtures.MonkeyPatch('ironic.common.utils.execute', fake_execute))
self.useFixture(fixtures.MonkeyPatch('os.rename', fake_rename))
self.useFixture(fixtures.MonkeyPatch('os.unlink', fake_unlink))
self.useFixture(
fixtures.MonkeyPatch('ironic.common.images.fetch',
lambda *_: None))
self.useFixture(
fixtures.MonkeyPatch('ironic.common.images.qemu_img_info',
fake_qemu_img_info))
self.useFixture(
fixtures.MonkeyPatch(
'ironic.openstack.common.fileutils.remove_path_on_error',
fake_rm_on_error))
self.useFixture(
fixtures.MonkeyPatch(
'ironic.openstack.common.fileutils.delete_if_exists',
fake_del_if_exists))
context = 'opaque context'
image_id = '4'
target = 't.qcow2'
self.executes = []
expected_commands = [('qemu-img', 'convert', '-O', 'raw',
't.qcow2.part', 't.qcow2.converted'),
('rm', 't.qcow2.part'),
('mv', 't.qcow2.converted', 't.qcow2')]
images.fetch_to_raw(context, image_id, target)
self.assertEqual(self.executes, expected_commands)
target = 't.raw'
self.executes = []
expected_commands = [('mv', 't.raw.part', 't.raw')]
images.fetch_to_raw(context, image_id, target)
self.assertEqual(self.executes, expected_commands)
target = 'backing.qcow2'
self.executes = []
expected_commands = [('rm', '-f', 'backing.qcow2.part')]
self.assertRaises(exception.ImageUnacceptable, images.fetch_to_raw,
context, image_id, target)
self.assertEqual(self.executes, expected_commands)
del self.executes
