def _download(self, image_href, data=None, method='data'):
"""Calls out to Glance for data and writes data.
:param image_href: The opaque image identifier.
:param data: (Optional) File object to write data to.
"""
image_id = service_utils.parse_image_id(image_href)
if (self.version == 2
and 'file' in CONF.glance.allowed_direct_url_schemes):
location = self._get_location(image_id)
url = urlparse.urlparse(location)
if url.scheme == "file":
with open(url.path, "r") as f:
filesize = os.path.getsize(f.name)
sendfile.sendfile(data.fileno(), f.fileno(), 0, filesize)
return
image_chunks = self.call(method, image_id)
# NOTE(dtantsur): when using Glance V2, image_chunks is a wrapper
# around real data, so we have to check the wrapped data for None.
# Glance V1 returns HTTP 404 in this case, so no need to fix it.
# TODO(dtantsur): remove the hasattr check when we no longer support
# Glance V1.
if hasattr(image_chunks, 'wrapped') and image_chunks.wrapped is None:
raise exception.ImageDownloadFailed(
image_href=image_href, reason=_('image contains no data.'))
if data is None:
return image_chunks
else:
for chunk in image_chunks:
data.write(chunk)
def download(self, image_href, data=None):
"""Calls out to Glance for data and writes data.
:param image_href: The opaque image identifier.
:param data: (Optional) File object to write data to.
"""
image_id = service_utils.parse_image_id(image_href)
if 'file' in CONF.glance.allowed_direct_url_schemes:
location = self._get_location(image_id)
url = urlparse.urlparse(location)
if url.scheme == "file":
with open(url.path, "r") as f:
filesize = os.path.getsize(f.name)
sendfile.sendfile(data.fileno(), f.fileno(), 0, filesize)
return
image_chunks = self.call('data', image_id)
# NOTE(dtantsur): when using Glance V2, image_chunks is a wrapper
# around real data, so we have to check the wrapped data for None.
if image_chunks.wrapped is None:
raise exception.ImageDownloadFailed(
image_href=image_href, reason=_('image contains no data.'))
if data is None:
return image_chunks
else:
for chunk in image_chunks:
data.write(chunk)
def _download(self, image_href, data=None, method='data'):
"""Calls out to Glance for data and writes data.
:param image_href: The opaque image identifier.
:param data: (Optional) File object to write data to.
"""
image_id = service_utils.parse_image_id(image_href)
if 'file' in CONF.glance.allowed_direct_url_schemes:
location = self._get_location(image_id)
url = urlparse.urlparse(location)
if url.scheme == "file":
with open(url.path, "r") as f:
filesize = os.path.getsize(f.name)
sendfile.sendfile(data.fileno(), f.fileno(), 0, filesize)
return
image_chunks = self.call(method, image_id)
# NOTE(dtantsur): when using Glance V2, image_chunks is a wrapper
# around real data, so we have to check the wrapped data for None.
if image_chunks.wrapped is None:
raise exception.ImageDownloadFailed(
image_href=image_href, reason=_('image contains no data.'))
if data is None:
return image_chunks
else:
for chunk in image_chunks:
data.write(chunk)
def _show(self, image_href, method='get'):
"""Returns a dict with image data for the given opaque image id.
:param image_href: The opaque image identifier.
:returns: A dict containing image metadata.
:raises: ImageNotFound
:raises: ImageUnacceptable if the image status is not active
"""
LOG.debug("Getting image metadata from glance. Image: %s",
image_href)
image_id = service_utils.parse_image_id(image_href)
image = self.call(method, image_id)
if not service_utils.is_image_active(image):
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("The image is required to be in an active state."))
if not service_utils.is_image_available(self.context, image):
raise exception.ImageNotFound(image_id=image_id)
base_image_meta = service_utils.translate_from_glance(image)
return base_image_meta
def _show(self, image_href, method='get'):
"""Returns a dict with image data for the given opaque image id.
:param image_href: The opaque image identifier.
:returns: A dict containing image metadata.
:raises: ImageNotFound
:raises: ImageUnacceptable if the image status is not active
"""
LOG.debug("Getting image metadata from glance. Image: %s",
image_href)
image_id = service_utils.parse_image_id(image_href)
image = self.call(method, image_id)
if not service_utils.is_image_active(image):
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("The image is required to be in an active state."))
if not service_utils.is_image_available(self.context, image):
raise exception.ImageNotFound(image_id=image_id)
base_image_meta = service_utils.translate_from_glance(image)
return base_image_meta
def test_parse_image_id_from_glance(self):
uuid = uuidutils.generate_uuid()
image_href = u'glance://some-stuff/%s' % uuid
parsed_id = service_utils.parse_image_id(image_href)
self.assertEqual(uuid, parsed_id)
def test_parse_image_id_from_uuid(self):
image_href = uuidutils.generate_uuid()
parsed_id = service_utils.parse_image_id(image_href)
self.assertEqual(image_href, parsed_id)
def fetch_image(self, href, dest_path, ctx=None, force_raw=True):
"""Fetch image by given href to the destination path.
Does nothing if destination path exists and is up to date with cache
and href contents.
Only creates a hard link (dest_path) to cached image if requested
image is already in cache and up to date with href contents.
Otherwise downloads an image, stores it in cache and creates a hard
link (dest_path) to it.
:param href: image UUID or href to fetch
:param dest_path: destination file path
:param ctx: context
:param force_raw: boolean value, whether to convert the image to raw
format
"""
img_download_lock_name = 'download-image'
if self.master_dir is None:
# NOTE(ghe): We don't share images between instances/hosts
if not CONF.parallel_image_downloads:
with lockutils.lock(img_download_lock_name):
_fetch(ctx, href, dest_path, force_raw)
else:
_fetch(ctx, href, dest_path, force_raw)
return
# TODO(ghe): have hard links and counts the same behaviour in all fs
# NOTE(vdrok): File name is converted to UUID if it's not UUID already,
# so that two images with same file names do not collide
if service_utils.is_glance_image(href):
master_file_name = service_utils.parse_image_id(href)
else:
master_file_name = str(uuid.uuid5(uuid.NAMESPACE_URL, href))
# NOTE(kaifeng) The ".converted" suffix acts as an indicator that the
# image cached has gone through the conversion logic.
if force_raw:
master_file_name = master_file_name + '.converted'
master_path = os.path.join(self.master_dir, master_file_name)
if CONF.parallel_image_downloads:
img_download_lock_name = 'download-image:%s' % master_file_name
# TODO(dtantsur): lock expiration time
with lockutils.lock(img_download_lock_name):
# NOTE(vdrok): After rebuild requested image can change, so we
# should ensure that dest_path and master_path (if exists) are
# pointing to the same file and their content is up to date
cache_up_to_date = _delete_master_path_if_stale(
master_path, href, ctx)
dest_up_to_date = _delete_dest_path_if_stale(
master_path, dest_path)
if cache_up_to_date and dest_up_to_date:
LOG.debug(
"Destination %(dest)s already exists "
"for image %(href)s", {
'href': href,
'dest': dest_path
})
return
if cache_up_to_date:
# NOTE(dtantsur): ensure we're not in the middle of clean up
with lockutils.lock('master_image'):
os.link(master_path, dest_path)
LOG.debug("Master cache hit for image %(href)s",
{'href': href})
return
LOG.info(
"Master cache miss for image %(href)s, "
"starting download", {'href': href})
self._download_image(href,
master_path,
dest_path,
ctx=ctx,
force_raw=force_raw)
# NOTE(dtantsur): we increased cache size - time to clean up
self.clean_up()
def fetch_image(self, href, dest_path, ctx=None, force_raw=True):
"""Fetch image by given href to the destination path.
Does nothing if destination path exists and is up to date with cache
and href contents.
Only creates a hard link (dest_path) to cached image if requested
image is already in cache and up to date with href contents.
Otherwise downloads an image, stores it in cache and creates a hard
link (dest_path) to it.
:param href: image UUID or href to fetch
:param dest_path: destination file path
:param ctx: context
:param force_raw: boolean value, whether to convert the image to raw
format
"""
img_download_lock_name = 'download-image'
if self.master_dir is None:
# NOTE(ghe): We don't share images between instances/hosts
if not CONF.parallel_image_downloads:
with lockutils.lock(img_download_lock_name):
_fetch(ctx, href, dest_path, force_raw)
else:
_fetch(ctx, href, dest_path, force_raw)
return
# TODO(ghe): have hard links and counts the same behaviour in all fs
# NOTE(vdrok): File name is converted to UUID if it's not UUID already,
# so that two images with same file names do not collide
if service_utils.is_glance_image(href):
master_file_name = service_utils.parse_image_id(href)
else:
# NOTE(vdrok): Doing conversion of href in case it's unicode
# string, UUID cannot be generated for unicode strings on python 2.
href_encoded = href.encode('utf-8') if six.PY2 else href
master_file_name = str(uuid.uuid5(uuid.NAMESPACE_URL,
href_encoded))
# NOTE(kaifeng) The ".converted" suffix acts as an indicator that the
# image cached has gone through the conversion logic.
if force_raw:
master_file_name = master_file_name + '.converted'
master_path = os.path.join(self.master_dir, master_file_name)
if CONF.parallel_image_downloads:
img_download_lock_name = 'download-image:%s' % master_file_name
# TODO(dtantsur): lock expiration time
with lockutils.lock(img_download_lock_name):
# NOTE(vdrok): After rebuild requested image can change, so we
# should ensure that dest_path and master_path (if exists) are
# pointing to the same file and their content is up to date
cache_up_to_date = _delete_master_path_if_stale(master_path, href,
ctx)
dest_up_to_date = _delete_dest_path_if_stale(master_path,
dest_path)
if cache_up_to_date and dest_up_to_date:
LOG.debug("Destination %(dest)s already exists "
"for image %(href)s",
{'href': href, 'dest': dest_path})
return
if cache_up_to_date:
# NOTE(dtantsur): ensure we're not in the middle of clean up
with lockutils.lock('master_image'):
os.link(master_path, dest_path)
LOG.debug("Master cache hit for image %(href)s",
{'href': href})
return
LOG.info("Master cache miss for image %(href)s, "
"starting download", {'href': href})
self._download_image(
href, master_path, dest_path, ctx=ctx, force_raw=force_raw)
# NOTE(dtantsur): we increased cache size - time to clean up
self.clean_up()
def test_parse_image_id_from_glance(self):
uuid = uuidutils.generate_uuid()
image_href = u'glance://some-stuff/%s' % uuid
parsed_id = service_utils.parse_image_id(image_href)
self.assertEqual(uuid, parsed_id)
def test_parse_image_id_from_uuid(self):
image_href = uuidutils.generate_uuid()
parsed_id = service_utils.parse_image_id(image_href)
self.assertEqual(image_href, parsed_id)
