def test__cache_instance_images_master_path(self): temp_dir = tempfile.mkdtemp() CONF.set_default('images_path', temp_dir, group='pxe') CONF.set_default('instance_master_path', os.path.join(temp_dir, 'instance_master_path'), group='pxe') fileutils.ensure_tree(CONF.pxe.instance_master_path) fd, tmp_master_image = tempfile.mkstemp( dir=CONF.pxe.instance_master_path) self.mox.StubOutWithMock(images, 'fetch_to_raw') self.mox.StubOutWithMock(tempfile, 'mkstemp') self.mox.StubOutWithMock(service_utils, 'parse_image_ref') tempfile.mkstemp(dir=CONF.pxe.instance_master_path).\ AndReturn((fd, tmp_master_image)) images.fetch_to_raw(None, 'glance://image_uuid', tmp_master_image, None).\ AndReturn(None) service_utils.parse_image_ref('glance://image_uuid').\ AndReturn(('image_uuid', None, None, None)) self.mox.ReplayAll() (uuid, image_path) = pxe._cache_instance_image(None, self.node) self.mox.VerifyAll() self.assertEqual(uuid, 'glance://image_uuid') self.assertEqual(image_path, temp_dir + '/fake_instance_name/disk')
def fetch_image(self, uuid, dest_path, ctx=None): """Fetch image with given uuid to the destination path. Does nothing if destination path exists. Only creates a link if master image for this UUID is already in cache. Otherwise downloads an image and also stores it in cache. :param uuid: image UUID or href to fetch :param dest_path: destination file path :param ctx: context """ img_download_lock_name = 'download-image' if self.master_dir is None: #NOTE(ghe): We don't share images between instances/hosts if not CONF.parallel_image_downloads: with lockutils.lock(img_download_lock_name, 'ironic-'): images.fetch_to_raw(ctx, uuid, dest_path, self._image_service) else: images.fetch_to_raw(ctx, uuid, dest_path, self._image_service) return #TODO(ghe): have hard links and counts the same behaviour in all fs master_file_name = service_utils.parse_image_ref(uuid)[0] master_path = os.path.join(self.master_dir, master_file_name) if CONF.parallel_image_downloads: img_download_lock_name = 'download-image:%s' % master_file_name # TODO(dtantsur): lock expiration time with lockutils.lock(img_download_lock_name, 'ironic-'): if os.path.exists(dest_path): LOG.debug("Destination %(dest)s already exists for " "image %(uuid)s" % { 'uuid': uuid, 'dest': dest_path }) return try: # NOTE(dtantsur): ensure we're not in the middle of clean up with lockutils.lock('master_image', 'ironic-'): os.link(master_path, dest_path) except OSError: LOG.info( _("Master cache miss for image %(uuid)s, " "starting download") % {'uuid': uuid}) else: LOG.debug("Master cache hit for image %(uuid)s", {'uuid': uuid}) return self._download_image(uuid, master_path, dest_path, ctx=ctx) # NOTE(dtantsur): we increased cache size - time to clean up self.clean_up()
def fetch_image(self, uuid, dest_path, ctx=None): """Fetch image with given uuid to the destination path. Does nothing if destination path exists. Only creates a link if master image for this UUID is already in cache. Otherwise downloads an image and also stores it in cache. :param uuid: image UUID or href to fetch :param dest_path: destination file path :param ctx: context """ img_download_lock_name = 'download-image' if self.master_dir is None: #NOTE(ghe): We don't share images between instances/hosts if not CONF.parallel_image_downloads: with lockutils.lock(img_download_lock_name, 'ironic-'): images.fetch_to_raw(ctx, uuid, dest_path, self._image_service) else: images.fetch_to_raw(ctx, uuid, dest_path, self._image_service) return #TODO(ghe): have hard links and counts the same behaviour in all fs master_file_name = service_utils.parse_image_ref(uuid)[0] master_path = os.path.join(self.master_dir, master_file_name) if CONF.parallel_image_downloads: img_download_lock_name = 'download-image:%s' % master_file_name # TODO(dtantsur): lock expiration time with lockutils.lock(img_download_lock_name, 'ironic-'): if os.path.exists(dest_path): LOG.debug("Destination %(dest)s already exists for " "image %(uuid)s" % {'uuid': uuid, 'dest': dest_path}) return try: # NOTE(dtantsur): ensure we're not in the middle of clean up with lockutils.lock('master_image', 'ironic-'): os.link(master_path, dest_path) except OSError: LOG.info(_("Master cache miss for image %(uuid)s, " "starting download") % {'uuid': uuid}) else: LOG.debug("Master cache hit for image %(uuid)s", {'uuid': uuid}) return self._download_image(uuid, master_path, dest_path, ctx=ctx) # NOTE(dtantsur): we increased cache size - time to clean up self.clean_up()
def test__cache_tftp_images_no_master_path(self): temp_dir = tempfile.mkdtemp() CONF.set_default('tftp_root', temp_dir, group='pxe') CONF.set_default('tftp_master_path', None, group='pxe') image_info = {'deploy_kernel': ['deploy_kernel', os.path.join(temp_dir, 'instance_uuid_123/deploy_kernel')]} self.mox.StubOutWithMock(images, 'fetch_to_raw') images.fetch_to_raw(None, 'deploy_kernel', os.path.join(temp_dir, 'instance_uuid_123/deploy_kernel'), None).AndReturn(None) self.mox.ReplayAll() pxe._cache_tftp_images(None, self.node, image_info) self.mox.VerifyAll()
def test__cache_instance_images_no_master_path(self): temp_dir = tempfile.mkdtemp() CONF.set_default('images_path', temp_dir, group='pxe') CONF.set_default('instance_master_path', None, group='pxe') self.mox.StubOutWithMock(images, 'fetch_to_raw') images.fetch_to_raw(None, 'glance://image_uuid', os.path.join(temp_dir, 'fake_instance_name/disk'), None).AndReturn(None) self.mox.ReplayAll() (uuid, image_path) = pxe._cache_instance_image(None, self.node) self.mox.VerifyAll() self.assertEqual(uuid, 'glance://image_uuid') self.assertEqual(image_path, os.path.join(temp_dir, 'fake_instance_name/disk'))
def test__cache_tftp_images_master_path(self): temp_dir = tempfile.mkdtemp() CONF.set_default('tftp_root', temp_dir, group='pxe') CONF.set_default('tftp_master_path', os.path.join(temp_dir, 'tftp_master_path'), group='pxe') image_info = {'deploy_kernel': ['deploy_kernel', temp_dir + '/instance_uuid_123/deploy_kernel']} fileutils.ensure_tree(CONF.pxe.tftp_master_path) fd, tmp_master_image = tempfile.mkstemp(dir=CONF.pxe.tftp_master_path) self.mox.StubOutWithMock(images, 'fetch_to_raw') self.mox.StubOutWithMock(tempfile, 'mkstemp') tempfile.mkstemp(dir=CONF.pxe.tftp_master_path).\ AndReturn((fd, tmp_master_image)) images.fetch_to_raw(None, 'deploy_kernel', tmp_master_image, None).\ AndReturn(None) self.mox.ReplayAll() pxe._cache_tftp_images(None, self.node, image_info) self.mox.VerifyAll()
def _download_image(self, uuid, master_path, dest_path, ctx=None): """Download image from Glance and store at a given path. This method should be called with uuid-specific lock taken. :param uuid: image UUID or href to fetch :param master_path: destination master path :param dest_path: destination file path :param ctx: context """ # TODO(ghe): timeout and retry for downloads # TODO(ghe): logging when image cannot be created fd, tmp_path = tempfile.mkstemp(dir=self.master_dir) os.close(fd) images.fetch_to_raw(ctx, uuid, tmp_path, self._image_service) # NOTE(dtantsur): no need for global lock here - master_path # will have link count >1 at any moment, so won't be cleaned up os.link(tmp_path, master_path) os.link(master_path, dest_path) os.unlink(tmp_path)
def _download_image(self, uuid, master_path, dest_path, ctx=None): """Download image from Glance and store at a given path. This method should be called with uuid-specific lock taken. :param uuid: image UUID or href to fetch :param master_path: destination master path :param dest_path: destination file path :param ctx: context """ #TODO(ghe): timeout and retry for downloads #TODO(ghe): logging when image cannot be created fd, tmp_path = tempfile.mkstemp(dir=self.master_dir) os.close(fd) images.fetch_to_raw(ctx, uuid, tmp_path, self._image_service) # NOTE(dtantsur): no need for global lock here - master_path # will have link count >1 at any moment, so won't be cleaned up os.link(tmp_path, master_path) os.link(master_path, dest_path) os.unlink(tmp_path)
def _get_image(ctx, path, uuid, master_path=None, image_service=None): #TODO(ghe): Revise this logic and cdocument process Bug #1199665 # When master_path defined, we save the images in this dir using the iamge # uuid as the file name. Deployments that use this images, creates a hard # link to keep track of this. When the link count of a master image is # equal to 1, can be deleted. #TODO(ghe): have hard links and count links the same behaviour in all fs #TODO(ghe): timeout and retry for downloads def _wait_for_download(): if not os.path.exists(lock_file): raise loopingcall.LoopingCallDone() # If the download of the image needed is in progress (lock file present) # we wait until the locks disappears and create the link. if master_path is None: #NOTE(ghe): We don't share images between instances/hosts images.fetch_to_raw(ctx, uuid, path, image_service) else: master_uuid = os.path.join(master_path, service_utils.parse_image_ref(uuid)[0]) lock_file = os.path.join(master_path, master_uuid + '.lock') _link_master_image(master_uuid, path) if not os.path.exists(path): fileutils.ensure_tree(master_path) if not _download_in_progress(lock_file): with fileutils.remove_path_on_error(lock_file): #TODO(ghe): logging when image cannot be created fd, tmp_path = tempfile.mkstemp(dir=master_path) os.close(fd) images.fetch_to_raw(ctx, uuid, tmp_path, image_service) _create_master_image(tmp_path, master_uuid, path) _remove_download_in_progress_lock(lock_file) else: #TODO(ghe): expiration time timer = loopingcall.FixedIntervalLoopingCall( _wait_for_download) timer.start(interval=1).wait() _link_master_image(master_uuid, path)
def test_fetch_raw_image(self): def fake_execute(*cmd, **kwargs): self.executes.append(cmd) return None, None def fake_rename(old, new): self.executes.append(('mv', old, new)) def fake_unlink(path): self.executes.append(('rm', path)) @contextlib.contextmanager def fake_rm_on_error(path): try: yield except Exception: with excutils.save_and_reraise_exception(): fake_del_if_exists(path) def fake_del_if_exists(path): self.executes.append(('rm', '-f', path)) def fake_qemu_img_info(path): class FakeImgInfo(object): pass file_format = path.split('.')[-1] if file_format == 'part': file_format = path.split('.')[-2] elif file_format == 'converted': file_format = 'raw' if 'backing' in path: backing_file = 'backing' else: backing_file = None FakeImgInfo.file_format = file_format FakeImgInfo.backing_file = backing_file return FakeImgInfo() self.useFixture(fixtures.MonkeyPatch( 'ironic.common.utils.execute', fake_execute)) self.useFixture(fixtures.MonkeyPatch('os.rename', fake_rename)) self.useFixture(fixtures.MonkeyPatch('os.unlink', fake_unlink)) self.useFixture(fixtures.MonkeyPatch( 'ironic.common.images.fetch', lambda *_: None)) self.useFixture(fixtures.MonkeyPatch( 'ironic.common.images.qemu_img_info', fake_qemu_img_info)) self.useFixture(fixtures.MonkeyPatch( 'ironic.openstack.common.fileutils.remove_path_on_error', fake_rm_on_error)) self.useFixture(fixtures.MonkeyPatch( 'ironic.openstack.common.fileutils.delete_if_exists', fake_del_if_exists)) context = 'opaque context' image_id = '4' target = 't.qcow2' self.executes = [] expected_commands = [('qemu-img', 'convert', '-O', 'raw', 't.qcow2.part', 't.qcow2.converted'), ('rm', 't.qcow2.part'), ('mv', 't.qcow2.converted', 't.qcow2')] images.fetch_to_raw(context, image_id, target) self.assertEqual(expected_commands, self.executes) target = 't.raw' self.executes = [] expected_commands = [('mv', 't.raw.part', 't.raw')] images.fetch_to_raw(context, image_id, target) self.assertEqual(expected_commands, self.executes) target = 'backing.qcow2' self.executes = [] expected_commands = [('rm', '-f', 'backing.qcow2.part')] self.assertRaises(exception.ImageUnacceptable, images.fetch_to_raw, context, image_id, target) self.assertEqual(expected_commands, self.executes) del self.executes
def test_fetch_raw_image(self): def fake_execute(*cmd, **kwargs): self.executes.append(cmd) return None, None def fake_rename(old, new): self.executes.append(('mv', old, new)) def fake_unlink(path): self.executes.append(('rm', path)) @contextlib.contextmanager def fake_rm_on_error(path): try: yield except Exception: with excutils.save_and_reraise_exception(): fake_del_if_exists(path) def fake_del_if_exists(path): self.executes.append(('rm', '-f', path)) def fake_qemu_img_info(path): class FakeImgInfo(object): pass file_format = path.split('.')[-1] if file_format == 'part': file_format = path.split('.')[-2] elif file_format == 'converted': file_format = 'raw' if 'backing' in path: backing_file = 'backing' else: backing_file = None FakeImgInfo.file_format = file_format FakeImgInfo.backing_file = backing_file return FakeImgInfo() self.useFixture( fixtures.MonkeyPatch('ironic.common.utils.execute', fake_execute)) self.useFixture(fixtures.MonkeyPatch('os.rename', fake_rename)) self.useFixture(fixtures.MonkeyPatch('os.unlink', fake_unlink)) self.useFixture( fixtures.MonkeyPatch('ironic.common.images.fetch', lambda *_: None)) self.useFixture( fixtures.MonkeyPatch('ironic.common.images.qemu_img_info', fake_qemu_img_info)) self.useFixture( fixtures.MonkeyPatch( 'ironic.openstack.common.fileutils.remove_path_on_error', fake_rm_on_error)) self.useFixture( fixtures.MonkeyPatch( 'ironic.openstack.common.fileutils.delete_if_exists', fake_del_if_exists)) context = 'opaque context' image_id = '4' target = 't.qcow2' self.executes = [] expected_commands = [('qemu-img', 'convert', '-O', 'raw', 't.qcow2.part', 't.qcow2.converted'), ('rm', 't.qcow2.part'), ('mv', 't.qcow2.converted', 't.qcow2')] images.fetch_to_raw(context, image_id, target) self.assertEqual(self.executes, expected_commands) target = 't.raw' self.executes = [] expected_commands = [('mv', 't.raw.part', 't.raw')] images.fetch_to_raw(context, image_id, target) self.assertEqual(self.executes, expected_commands) target = 'backing.qcow2' self.executes = [] expected_commands = [('rm', '-f', 'backing.qcow2.part')] self.assertRaises(exception.ImageUnacceptable, images.fetch_to_raw, context, image_id, target) self.assertEqual(self.executes, expected_commands) del self.executes