def _download(self, image_href, data=None, method='data'): """Calls out to Glance for data and writes data. :param image_href: The opaque image identifier. :param data: (Optional) File object to write data to. """ image_id = service_utils.parse_image_id(image_href) if (self.version == 2 and 'file' in CONF.glance.allowed_direct_url_schemes): location = self._get_location(image_id) url = urlparse.urlparse(location) if url.scheme == "file": with open(url.path, "r") as f: filesize = os.path.getsize(f.name) sendfile.sendfile(data.fileno(), f.fileno(), 0, filesize) return image_chunks = self.call(method, image_id) # NOTE(dtantsur): when using Glance V2, image_chunks is a wrapper # around real data, so we have to check the wrapped data for None. # Glance V1 returns HTTP 404 in this case, so no need to fix it. # TODO(dtantsur): remove the hasattr check when we no longer support # Glance V1. if hasattr(image_chunks, 'wrapped') and image_chunks.wrapped is None: raise exception.ImageDownloadFailed( image_href=image_href, reason=_('image contains no data.')) if data is None: return image_chunks else: for chunk in image_chunks: data.write(chunk)
def download(self, image_href, data=None): """Calls out to Glance for data and writes data. :param image_href: The opaque image identifier. :param data: (Optional) File object to write data to. """ image_id = service_utils.parse_image_id(image_href) if 'file' in CONF.glance.allowed_direct_url_schemes: location = self._get_location(image_id) url = urlparse.urlparse(location) if url.scheme == "file": with open(url.path, "r") as f: filesize = os.path.getsize(f.name) sendfile.sendfile(data.fileno(), f.fileno(), 0, filesize) return image_chunks = self.call('data', image_id) # NOTE(dtantsur): when using Glance V2, image_chunks is a wrapper # around real data, so we have to check the wrapped data for None. if image_chunks.wrapped is None: raise exception.ImageDownloadFailed( image_href=image_href, reason=_('image contains no data.')) if data is None: return image_chunks else: for chunk in image_chunks: data.write(chunk)
def _download(self, image_href, data=None, method='data'): """Calls out to Glance for data and writes data. :param image_href: The opaque image identifier. :param data: (Optional) File object to write data to. """ image_id = service_utils.parse_image_id(image_href) if 'file' in CONF.glance.allowed_direct_url_schemes: location = self._get_location(image_id) url = urlparse.urlparse(location) if url.scheme == "file": with open(url.path, "r") as f: filesize = os.path.getsize(f.name) sendfile.sendfile(data.fileno(), f.fileno(), 0, filesize) return image_chunks = self.call(method, image_id) # NOTE(dtantsur): when using Glance V2, image_chunks is a wrapper # around real data, so we have to check the wrapped data for None. if image_chunks.wrapped is None: raise exception.ImageDownloadFailed( image_href=image_href, reason=_('image contains no data.')) if data is None: return image_chunks else: for chunk in image_chunks: data.write(chunk)
def _show(self, image_href, method='get'): """Returns a dict with image data for the given opaque image id. :param image_href: The opaque image identifier. :returns: A dict containing image metadata. :raises: ImageNotFound :raises: ImageUnacceptable if the image status is not active """ LOG.debug("Getting image metadata from glance. Image: %s", image_href) image_id = service_utils.parse_image_id(image_href) image = self.call(method, image_id) if not service_utils.is_image_active(image): raise exception.ImageUnacceptable( image_id=image_id, reason=_("The image is required to be in an active state.")) if not service_utils.is_image_available(self.context, image): raise exception.ImageNotFound(image_id=image_id) base_image_meta = service_utils.translate_from_glance(image) return base_image_meta
def _show(self, image_href, method='get'): """Returns a dict with image data for the given opaque image id. :param image_href: The opaque image identifier. :returns: A dict containing image metadata. :raises: ImageNotFound :raises: ImageUnacceptable if the image status is not active """ LOG.debug("Getting image metadata from glance. Image: %s", image_href) image_id = service_utils.parse_image_id(image_href) image = self.call(method, image_id) if not service_utils.is_image_active(image): raise exception.ImageUnacceptable( image_id=image_id, reason=_("The image is required to be in an active state.")) if not service_utils.is_image_available(self.context, image): raise exception.ImageNotFound(image_id=image_id) base_image_meta = service_utils.translate_from_glance(image) return base_image_meta
def test_parse_image_id_from_glance(self): uuid = uuidutils.generate_uuid() image_href = u'glance://some-stuff/%s' % uuid parsed_id = service_utils.parse_image_id(image_href) self.assertEqual(uuid, parsed_id)
def test_parse_image_id_from_uuid(self): image_href = uuidutils.generate_uuid() parsed_id = service_utils.parse_image_id(image_href) self.assertEqual(image_href, parsed_id)
def fetch_image(self, href, dest_path, ctx=None, force_raw=True): """Fetch image by given href to the destination path. Does nothing if destination path exists and is up to date with cache and href contents. Only creates a hard link (dest_path) to cached image if requested image is already in cache and up to date with href contents. Otherwise downloads an image, stores it in cache and creates a hard link (dest_path) to it. :param href: image UUID or href to fetch :param dest_path: destination file path :param ctx: context :param force_raw: boolean value, whether to convert the image to raw format """ img_download_lock_name = 'download-image' if self.master_dir is None: # NOTE(ghe): We don't share images between instances/hosts if not CONF.parallel_image_downloads: with lockutils.lock(img_download_lock_name): _fetch(ctx, href, dest_path, force_raw) else: _fetch(ctx, href, dest_path, force_raw) return # TODO(ghe): have hard links and counts the same behaviour in all fs # NOTE(vdrok): File name is converted to UUID if it's not UUID already, # so that two images with same file names do not collide if service_utils.is_glance_image(href): master_file_name = service_utils.parse_image_id(href) else: master_file_name = str(uuid.uuid5(uuid.NAMESPACE_URL, href)) # NOTE(kaifeng) The ".converted" suffix acts as an indicator that the # image cached has gone through the conversion logic. if force_raw: master_file_name = master_file_name + '.converted' master_path = os.path.join(self.master_dir, master_file_name) if CONF.parallel_image_downloads: img_download_lock_name = 'download-image:%s' % master_file_name # TODO(dtantsur): lock expiration time with lockutils.lock(img_download_lock_name): # NOTE(vdrok): After rebuild requested image can change, so we # should ensure that dest_path and master_path (if exists) are # pointing to the same file and their content is up to date cache_up_to_date = _delete_master_path_if_stale( master_path, href, ctx) dest_up_to_date = _delete_dest_path_if_stale( master_path, dest_path) if cache_up_to_date and dest_up_to_date: LOG.debug( "Destination %(dest)s already exists " "for image %(href)s", { 'href': href, 'dest': dest_path }) return if cache_up_to_date: # NOTE(dtantsur): ensure we're not in the middle of clean up with lockutils.lock('master_image'): os.link(master_path, dest_path) LOG.debug("Master cache hit for image %(href)s", {'href': href}) return LOG.info( "Master cache miss for image %(href)s, " "starting download", {'href': href}) self._download_image(href, master_path, dest_path, ctx=ctx, force_raw=force_raw) # NOTE(dtantsur): we increased cache size - time to clean up self.clean_up()
def fetch_image(self, href, dest_path, ctx=None, force_raw=True): """Fetch image by given href to the destination path. Does nothing if destination path exists and is up to date with cache and href contents. Only creates a hard link (dest_path) to cached image if requested image is already in cache and up to date with href contents. Otherwise downloads an image, stores it in cache and creates a hard link (dest_path) to it. :param href: image UUID or href to fetch :param dest_path: destination file path :param ctx: context :param force_raw: boolean value, whether to convert the image to raw format """ img_download_lock_name = 'download-image' if self.master_dir is None: # NOTE(ghe): We don't share images between instances/hosts if not CONF.parallel_image_downloads: with lockutils.lock(img_download_lock_name): _fetch(ctx, href, dest_path, force_raw) else: _fetch(ctx, href, dest_path, force_raw) return # TODO(ghe): have hard links and counts the same behaviour in all fs # NOTE(vdrok): File name is converted to UUID if it's not UUID already, # so that two images with same file names do not collide if service_utils.is_glance_image(href): master_file_name = service_utils.parse_image_id(href) else: # NOTE(vdrok): Doing conversion of href in case it's unicode # string, UUID cannot be generated for unicode strings on python 2. href_encoded = href.encode('utf-8') if six.PY2 else href master_file_name = str(uuid.uuid5(uuid.NAMESPACE_URL, href_encoded)) # NOTE(kaifeng) The ".converted" suffix acts as an indicator that the # image cached has gone through the conversion logic. if force_raw: master_file_name = master_file_name + '.converted' master_path = os.path.join(self.master_dir, master_file_name) if CONF.parallel_image_downloads: img_download_lock_name = 'download-image:%s' % master_file_name # TODO(dtantsur): lock expiration time with lockutils.lock(img_download_lock_name): # NOTE(vdrok): After rebuild requested image can change, so we # should ensure that dest_path and master_path (if exists) are # pointing to the same file and their content is up to date cache_up_to_date = _delete_master_path_if_stale(master_path, href, ctx) dest_up_to_date = _delete_dest_path_if_stale(master_path, dest_path) if cache_up_to_date and dest_up_to_date: LOG.debug("Destination %(dest)s already exists " "for image %(href)s", {'href': href, 'dest': dest_path}) return if cache_up_to_date: # NOTE(dtantsur): ensure we're not in the middle of clean up with lockutils.lock('master_image'): os.link(master_path, dest_path) LOG.debug("Master cache hit for image %(href)s", {'href': href}) return LOG.info("Master cache miss for image %(href)s, " "starting download", {'href': href}) self._download_image( href, master_path, dest_path, ctx=ctx, force_raw=force_raw) # NOTE(dtantsur): we increased cache size - time to clean up self.clean_up()
def test_parse_image_id_from_glance(self): uuid = uuidutils.generate_uuid() image_href = u'glance://some-stuff/%s' % uuid parsed_id = service_utils.parse_image_id(image_href) self.assertEqual(uuid, parsed_id)
def test_parse_image_id_from_uuid(self): image_href = uuidutils.generate_uuid() parsed_id = service_utils.parse_image_id(image_href) self.assertEqual(image_href, parsed_id)