def _delete_image(self,
target_pool,
image_name,
snapshot_name=None,
context=None):
"""
Delete RBD image and snapshot.
:param image_name: Image's name
:param snapshot_name: Image snapshot's name
:raises: NotFound if image does not exist;
InUseByStore if image is in use or snapshot unprotect failed
"""
with self.get_connection(conffile=self.conf_file,
rados_id=self.user) as conn:
with conn.open_ioctx(target_pool) as ioctx:
try:
# First remove snapshot.
if snapshot_name is not None:
with rbd.Image(ioctx, image_name) as image:
try:
self._unprotect_snapshot(image, snapshot_name)
image.remove_snap(snapshot_name)
except rbd.ImageNotFound as exc:
msg = (_("Snap Operating Exception "
"%(snap_exc)s "
"Snapshot does not exist.") % {
'snap_exc': exc
})
LOG.debug(msg)
except rbd.ImageBusy as exc:
log_msg = (_LE("Snap Operating Exception "
"%(snap_exc)s "
"Snapshot is in use.") % {
'snap_exc': exc
})
LOG.error(log_msg)
raise exceptions.InUseByStore()
# Then delete image.
rbd.RBD().remove(ioctx, image_name)
except rbd.ImageHasSnapshots:
log_msg = (_LE("Remove image %(img_name)s failed. "
"It has snapshot(s) left.") % {
'img_name': image_name
})
LOG.error(log_msg)
raise exceptions.HasSnapshot()
except rbd.ImageBusy:
log_msg = (_LE("Remove image %(img_name)s failed. "
"It is in use.") % {
'img_name': image_name
})
LOG.error(log_msg)
raise exceptions.InUseByStore()
except rbd.ImageNotFound:
msg = _("RBD image %s does not exist") % image_name
raise exceptions.NotFound(message=msg)
def _open_cinder_volume(self, client, volume, mode):
attach_mode = 'rw' if mode == 'wb' else 'ro'
device = None
root_helper = get_root_helper()
host = socket.gethostname()
properties = connector.get_connector_properties(root_helper, host,
False, False)
try:
volume.reserve(volume)
except cinder_exception.ClientException as e:
msg = (_('Failed to reserve volume %(volume_id)s: %(error)s')
% {'volume_id': volume.id, 'error': e})
LOG.error(msg)
raise exceptions.BackendException(msg)
try:
connection_info = volume.initialize_connection(volume, properties)
conn = connector.InitiatorConnector.factory(
connection_info['driver_volume_type'], root_helper,
conn=connection_info)
device = conn.connect_volume(connection_info['data'])
volume.attach(None, None, attach_mode, host_name=host)
volume = self._wait_volume_status(volume, 'attaching', 'in-use')
LOG.debug('Opening host device "%s"', device['path'])
with temporary_chown(device['path']), \
open(device['path'], mode) as f:
yield f
except Exception:
LOG.exception(_LE('Exception while accessing to cinder volume '
'%(volume_id)s.'), {'volume_id': volume.id})
raise
finally:
if volume.status == 'in-use':
volume.begin_detaching(volume)
elif volume.status == 'attaching':
volume.unreserve(volume)
if device:
try:
conn.disconnect_volume(connection_info['data'], device)
except Exception:
LOG.exception(_LE('Failed to disconnect volume '
'%(volume_id)s.'),
{'volume_id': volume.id})
try:
volume.terminate_connection(volume, properties)
except Exception:
LOG.exception(_LE('Failed to terminate connection of volume '
'%(volume_id)s.'), {'volume_id': volume.id})
try:
client.volumes.detach(volume)
except Exception:
LOG.exception(_LE('Failed to detach volume %(volume_id)s.'),
{'volume_id': volume.id})
def _open_cinder_volume(self, client, volume, mode):
attach_mode = 'rw' if mode == 'wb' else 'ro'
device = None
root_helper = get_root_helper()
host = socket.gethostname()
properties = connector.get_connector_properties(root_helper, host,
False, False)
try:
volume.reserve(volume)
except cinder_exception.ClientException as e:
msg = (_('Failed to reserve volume %(volume_id)s: %(error)s')
% {'volume_id': volume.id, 'error': e})
LOG.error(msg)
raise exceptions.BackendException(msg)
try:
connection_info = volume.initialize_connection(volume, properties)
conn = connector.InitiatorConnector.factory(
connection_info['driver_volume_type'], root_helper,
conn=connection_info)
device = conn.connect_volume(connection_info['data'])
volume.attach(None, None, attach_mode, host_name=host)
volume = self._wait_volume_status(volume, 'attaching', 'in-use')
LOG.debug('Opening host device "%s"', device['path'])
with temporary_chown(device['path']), \
open(device['path'], mode) as f:
yield f
except Exception:
LOG.exception(_LE('Exception while accessing to cinder volume '
'%(volume_id)s.'), {'volume_id': volume.id})
raise
finally:
if volume.status == 'in-use':
volume.begin_detaching(volume)
elif volume.status == 'attaching':
volume.unreserve(volume)
if device:
try:
conn.disconnect_volume(connection_info['data'], device)
except Exception:
LOG.exception(_LE('Failed to disconnect volume '
'%(volume_id)s.'),
{'volume_id': volume.id})
try:
volume.terminate_connection(volume, properties)
except Exception:
LOG.exception(_LE('Failed to terminate connection of volume '
'%(volume_id)s.'), {'volume_id': volume.id})
try:
client.volumes.detach(volume)
except Exception:
LOG.exception(_LE('Failed to detach volume %(volume_id)s.'),
{'volume_id': volume.id})
def add(self, image_id, image_file, image_size, context=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exceptions.Duplicate` if the image already
existed
`glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
loc = StoreLocation(
{
"scheme": self.scheme,
"server_host": self.server_host,
"image_dir": self.store_image_dir,
"datacenter_path": self.datacenter_path,
"datastore_name": self.datastore_name,
"image_id": image_id,
}
)
# NOTE(arnaud): use a decorator when the config is not tied to self
for i in range(self.api_retry_count + 1):
cookie = self._build_vim_cookie_header(self._session.vim.client.options.transport.cookiejar)
headers = {"Connection": "Keep-Alive", "Cookie": cookie, "Transfer-Encoding": "chunked"}
try:
conn = self._get_http_conn("PUT", loc, headers, content=image_file)
res = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to upload content of image " "%(image)s"), {"image": image_id})
if res.status == httplib.UNAUTHORIZED:
self._create_session()
image_file.rewind()
continue
if res.status == httplib.CONFLICT:
raise exceptions.Duplicate(_("Image file %(image_id)s already " "exists!") % {"image_id": image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = _LE("Failed to upload content of image %(image)s") % {"image": image_id}
LOG.error(msg)
raise exceptions.UnexpectedStatus(status=res.status, body=res.read())
break
return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def _delete_image(self, target_pool, image_name,
snapshot_name=None, context=None):
"""
Delete RBD image and snapshot.
:param image_name: Image's name
:param snapshot_name: Image snapshot's name
:raises: NotFound if image does not exist;
InUseByStore if image is in use or snapshot unprotect failed
"""
with self.get_connection(conffile=self.conf_file,
rados_id=self.user) as conn:
with conn.open_ioctx(target_pool) as ioctx:
try:
# First remove snapshot.
if snapshot_name is not None:
with rbd.Image(ioctx, image_name) as image:
try:
self._unprotect_snapshot(image, snapshot_name)
image.remove_snap(snapshot_name)
except rbd.ImageNotFound as exc:
msg = (_("Snap Operating Exception "
"%(snap_exc)s "
"Snapshot does not exist.") %
{'snap_exc': exc})
LOG.debug(msg)
except rbd.ImageBusy as exc:
log_msg = (_LE("Snap Operating Exception "
"%(snap_exc)s "
"Snapshot is in use.") %
{'snap_exc': exc})
LOG.error(log_msg)
raise exceptions.InUseByStore()
# Then delete image.
rbd.RBD().remove(ioctx, image_name)
except rbd.ImageHasSnapshots:
log_msg = (_LE("Remove image %(img_name)s failed. "
"It has snapshot(s) left.") %
{'img_name': image_name})
LOG.error(log_msg)
raise exceptions.HasSnapshot()
except rbd.ImageBusy:
log_msg = (_LE("Remove image %(img_name)s failed. "
"It is in use.") %
{'img_name': image_name})
LOG.error(log_msg)
raise exceptions.InUseByStore()
except rbd.ImageNotFound:
msg = _("RBD image %s does not exist") % image_name
raise exceptions.NotFound(message=msg)
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file and returns the image size
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises: `glance_store.exceptions.NotFound` if image does not exist
:rtype int
"""
loc = location.store_location
try:
self._check_context(context)
volume = get_cinderclient(self.conf,
context).volumes.get(loc.volume_id)
return int(volume.metadata.get('image_size',
volume.size * units.Gi))
except cinder_exception.NotFound:
raise exceptions.NotFound(image=loc.volume_id)
except Exception:
LOG.exception(_LE("Failed to get image size due to "
"internal error."))
return 0
def _get_metadata(self, filepath):
"""Return metadata dictionary.
If metadata is provided as list of dictionaries then return
metadata as dictionary containing 'id' and 'mountpoint'.
If there are multiple nfs directories (mountpoints) configured
for glance, then we need to create metadata JSON file as list
of dictionaries containing all mountpoints with unique id.
But Nova will not be able to find in which directory (mountpoint)
image is present if we store list of dictionary(containing mountpoints)
in glance image metadata. So if there are multiple mountpoints then
we will return dict containing exact mountpoint where image is stored.
If image path does not start with any of the 'mountpoint' provided
in metadata JSON file then error is logged and empty
dictionary is returned.
:param filepath: Path of image on store
:returns: metadata dictionary
"""
if self.FILESYSTEM_STORE_METADATA:
for image_meta in self.FILESYSTEM_STORE_METADATA:
if filepath.startswith(image_meta['mountpoint']):
return image_meta
reason = (_LE("The image path %(path)s does not match with "
"any of the mountpoint defined in "
"metadata: %(metadata)s. An empty dictionary "
"will be returned to the client.")
% dict(path=filepath,
metadata=self.FILESYSTEM_STORE_METADATA))
LOG.error(reason)
return {}
def delete(self, location, context=None):
"""Takes a `glance_store.location.Location` object that indicates
where to find the image file to delete
:location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
file_path = "[%s] %s" % (self.datastore_name, location.store_location.path[len(DS_URL_PREFIX) :])
search_index_moref = self._service_content.searchIndex
dc_moref = self._session.invoke_api(
self._session.vim, "FindByInventoryPath", search_index_moref, inventoryPath=self.datacenter_path
)
delete_task = self._session.invoke_api(
self._session.vim,
"DeleteDatastoreFile_Task",
self._service_content.fileManager,
name=file_path,
datacenter=dc_moref,
)
try:
self._session.wait_for_task(delete_task)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to delete image %(image)s " "content.") % {"image": location.image_id})
def _get_metadata(self, filepath):
"""Return metadata dictionary.
If metadata is provided as list of dictionaries then return
metadata as dictionary containing 'id' and 'mountpoint'.
If there are multiple nfs directories (mountpoints) configured
for glance, then we need to create metadata JSON file as list
of dictionaries containing all mountpoints with unique id.
But Nova will not be able to find in which directory (mountpoint)
image is present if we store list of dictionary(containing mountpoints)
in glance image metadata. So if there are multiple mountpoints then
we will return dict containing exact mountpoint where image is stored.
If image path does not start with any of the 'mountpoint' provided
in metadata JSON file then error is logged and empty
dictionary is returned.
:param filepath: Path of image on store
:returns: metadata dictionary
"""
if self.FILESYSTEM_STORE_METADATA:
for image_meta in self.FILESYSTEM_STORE_METADATA:
if filepath.startswith(image_meta['mountpoint']):
return image_meta
reason = (_LE("The image path %(path)s does not match with "
"any of the mountpoint defined in "
"metadata: %(metadata)s. An empty dictionary "
"will be returned to the client.")
% dict(path=filepath,
metadata=self.FILESYSTEM_STORE_METADATA))
LOG.error(reason)
return {}
def delete(self, location, context=None):
"""Takes a `glance_store.location.Location` object that indicates
where to find the image file to delete
:location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
file_path = '[%s] %s' % (
location.store_location.datastore_name,
location.store_location.path[len(DS_URL_PREFIX):])
dc_obj = self._get_datacenter(location.store_location.datacenter_path)
delete_task = self.session.invoke_api(
self.session.vim,
'DeleteDatastoreFile_Task',
self.session.vim.service_content.fileManager,
name=file_path,
datacenter=dc_obj.ref)
try:
self.session.wait_for_task(delete_task)
except vexc.FileNotFoundException:
msg = _('Image file %s not found') % file_path
LOG.warn(msg)
raise exceptions.NotFound(message=msg)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to delete image %(image)s '
'content.') % {'image': location.image_id})
def delete(self, location, context=None):
"""Takes a `glance_store.location.Location` object that indicates
where to find the image file to delete
:location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
file_path = '[%s] %s' % (self.datastore_name, location.store_location.
path[len(DS_URL_PREFIX):])
search_index_moref = self._service_content.searchIndex
dc_moref = self._session.invoke_api(self._session.vim,
'FindByInventoryPath',
search_index_moref,
inventoryPath=self.datacenter_path)
delete_task = self._session.invoke_api(
self._session.vim,
'DeleteDatastoreFile_Task',
self._service_content.fileManager,
name=file_path,
datacenter=dc_moref)
try:
self._session.wait_for_task(delete_task)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_LE('Failed to delete image %(image)s '
'content.') % {'image': location.image_id})
def delete(self, location, context=None):
"""Takes a `glance_store.location.Location` object that indicates
where to find the image file to delete
:location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
file_path = '[%s] %s' % (
location.store_location.datastore_name,
location.store_location.path[len(DS_URL_PREFIX):])
dc_obj = self._get_datacenter(location.store_location.datacenter_path)
delete_task = self.session.invoke_api(
self.session.vim,
'DeleteDatastoreFile_Task',
self.session.vim.service_content.fileManager,
name=file_path,
datacenter=dc_obj.ref)
try:
self.session.wait_for_task(delete_task)
except vexc.FileNotFoundException:
msg = _('Image file %s not found') % file_path
LOG.warn(msg)
raise exceptions.NotFound(message=msg)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_LE('Failed to delete image %(image)s '
'content.') % {'image': location.image_id})
def get(self, location, offset=0, chunk_size=None, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
"""
try:
conn, resp, content_length = self._query(location, 'GET')
except socket.error:
reason = _LE("Remote server where the image is present "
"is unavailable.")
LOG.error(reason)
raise exceptions.RemoteServiceUnavailable()
iterator = http_response_iterator(conn, resp, self.READ_CHUNKSIZE)
class ResponseIndexable(glance_store.Indexable):
def another(self):
try:
return self.wrapped.next()
except StopIteration:
return ''
return (ResponseIndexable(iterator, content_length), content_length)
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file and returns the image size
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises: `glance_store.exceptions.NotFound` if image does not exist
:rtype: int
"""
loc = location.store_location
try:
self._check_context(context)
volume = self.get_cinderclient(context).volumes.get(loc.volume_id)
return int(
volume.metadata.get('image_size', volume.size * units.Gi))
except cinder_exception.NotFound:
raise exceptions.NotFound(image=loc.volume_id)
except Exception:
LOG.exception(
_LE("Failed to get image size due to "
"internal error."))
return 0
def get(self, location, offset=0, chunk_size=None, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
"""
try:
conn, resp, content_length = self._query(location, 'GET')
except socket.error:
reason = _LE("Remote server where the image is present "
"is unavailable.")
LOG.error(reason)
raise exceptions.RemoteServiceUnavailable()
iterator = http_response_iterator(conn, resp, self.READ_CHUNKSIZE)
class ResponseIndexable(glance_store.Indexable):
def another(self):
try:
return next(self.wrapped)
except StopIteration:
return ''
return (ResponseIndexable(iterator, content_length), content_length)
def rewind(self):
try:
self.data.seek(0)
self._size = 0
self.checksum = hashlib.md5()
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to rewind image content'))
def rewind(self):
try:
self.data.seek(0)
self._size = 0
self.checksum = hashlib.md5()
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to rewind image content'))
def _query(self, location, method, depth=0):
if depth > MAX_REDIRECTS:
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.", {
'max_redirects': MAX_REDIRECTS
})
LOG.debug(msg)
raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
loc = location.store_location
# NOTE(arnaud): use a decorator when the config is not tied to self
for i in range(self.api_retry_count + 1):
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Cookie': cookie}
try:
conn = self._get_http_conn(method, loc, headers)
resp = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_LE('Failed to access image %(image)s '
'content.') % {'image': location.image_id})
if resp.status >= 400:
if resp.status == httplib.UNAUTHORIZED:
self._create_session()
continue
if resp.status == httplib.NOT_FOUND:
reason = _('VMware datastore could not find image at URI.')
LOG.info(reason)
raise exceptions.NotFound(message=reason)
msg = ('HTTP request returned a %(status)s status code.' % {
'status': resp.status
})
LOG.debug(msg)
raise exceptions.BadStoreUri(msg)
break
location_header = resp.getheader('location')
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.") % {
'path': loc.path,
'status': resp.status
})
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
location_class = glance_store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, method, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def get_connection(self, conffile, rados_id):
client = rados.Rados(conffile=conffile, rados_id=rados_id)
try:
client.connect(timeout=self.connect_timeout)
except rados.Error:
msg = _LE("Error connecting to ceph cluster.")
LOG.exception(msg)
raise exceptions.BackendException()
try:
yield client
finally:
client.shutdown()
def get_connection(self, conffile, rados_id):
client = rados.Rados(conffile=conffile, rados_id=rados_id)
try:
client.connect(timeout=self.connect_timeout)
except rados.Error:
msg = _LE("Error connecting to ceph cluster.")
LOG.exception(msg)
raise exceptions.BackendException()
try:
yield client
finally:
client.shutdown()
def _query(self, location, method, depth=0):
if depth > MAX_REDIRECTS:
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.", {'max_redirects': MAX_REDIRECTS})
LOG.debug(msg)
raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
loc = location.store_location
# NOTE(arnaud): use a decorator when the config is not tied to self
for i in range(self.api_retry_count + 1):
cookie = self._build_vim_cookie_header(
self.session.vim.client.options.transport.cookiejar)
headers = {'Cookie': cookie}
try:
conn = self._get_http_conn(method, loc, headers)
resp = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to access image %(image)s '
'content.') % {'image':
location.image_id})
if resp.status >= 400:
if resp.status == httplib.UNAUTHORIZED:
self.reset_session(force=True)
continue
if resp.status == httplib.NOT_FOUND:
reason = _('VMware datastore could not find image at URI.')
LOG.info(reason)
raise exceptions.NotFound(message=reason)
msg = ('HTTP request returned a %(status)s status code.'
% {'status': resp.status})
LOG.debug(msg)
raise exceptions.BadStoreUri(msg)
break
location_header = resp.getheader('location')
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.")
% {'path': loc.path, 'status': resp.status})
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
location_class = glance_store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, method, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def _real_umount(self, mountpoint, rootwrap_helper):
# Unmount and delete a mountpoint.
# Return mount state after umount (i.e. True means still mounted)
LOG.debug('Unmounting %(mountpoint)s', {'mountpoint': mountpoint})
try:
processutils.execute('umount', mountpoint, run_as_root=True,
attempts=3, delay_on_retry=True,
root_helper=rootwrap_helper)
except processutils.ProcessExecutionError as ex:
LOG.error(_LE("Couldn't unmount %(mountpoint)s: %(reason)s"),
{'mountpoint': mountpoint, 'reason': ex})
if not os.path.ismount(mountpoint):
try:
os.rmdir(mountpoint)
except Exception as ex:
LOG.error(_LE("Couldn't remove directory %(mountpoint)s: "
"%(reason)s"),
{'mountpoint': mountpoint,
'reason': ex})
return False
return True
def _load_config(self):
if self.backend_group:
scf = getattr(self.conf,
self.backend_group).swift_store_config_file
else:
scf = self.conf.glance_store.swift_store_config_file
try:
conf_file = self.conf.find_file(scf)
CONFIG.read(conf_file)
except Exception as e:
msg = (_("swift config file "
"%(conf)s:%(exc)s not found"),
{'conf': scf,
'exc': e})
LOG.error(msg)
raise exceptions.BadStoreConfiguration(store_name='swift',
reason=msg)
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
for param in ('auth_address',
'user',
'key',
'project_domain_id',
'project_domain_name',
'user_domain_id',
'user_domain_name'):
reference[param] = CONFIG.get(ref, param)
try:
reference['auth_version'] = CONFIG.get(ref, 'auth_version')
except configparser.NoOptionError:
if self.backend_group:
av = getattr(
self.conf,
self.backend_group).swift_store_auth_version
else:
av = self.conf.glance_store.swift_store_auth_version
reference['auth_version'] = av
account_params[ref] = reference
except (ValueError, SyntaxError, configparser.NoOptionError) as e:
LOG.exception(_LE("Invalid format of swift store config cfg"))
return account_params
def select_datastore(self, image_size):
"""Select a datastore with free space larger than image size."""
for k, v in sorted(six.iteritems(self.datastores), reverse=True):
max_ds = None
max_fs = 0
for ds in v:
# Update with current freespace
ds.freespace = self._get_freespace(ds)
if ds.freespace > max_fs:
max_ds = ds
max_fs = ds.freespace
if max_ds and max_ds.freespace >= image_size:
return max_ds
msg = _LE("No datastore found with enough free space to contain an "
"image of size %d") % image_size
LOG.error(msg)
raise exceptions.StorageFull()
def select_datastore(self, image_size):
"""Select a datastore with free space larger than image size."""
for k, v in sorted(six.iteritems(self.datastores), reverse=True):
max_ds = None
max_fs = 0
for ds in v:
# Update with current freespace
ds.freespace = self._get_freespace(ds)
if ds.freespace > max_fs:
max_ds = ds
max_fs = ds.freespace
if max_ds and max_ds.freespace >= image_size:
return max_ds
msg = _LE("No datastore found with enough free space to contain an "
"image of size %d") % image_size
LOG.error(msg)
raise exceptions.StorageFull()
def _load_config(self):
if self.backend_group:
scf = getattr(self.conf,
self.backend_group).swift_store_config_file
else:
scf = self.conf.glance_store.swift_store_config_file
try:
conf_file = self.conf.find_file(scf)
CONFIG.read(conf_file)
except Exception as e:
msg = (_("swift config file "
"%(conf)s:%(exc)s not found"), {
'conf': scf,
'exc': e
})
LOG.error(msg)
raise exceptions.BadStoreConfiguration(store_name='swift',
reason=msg)
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
for param in ('auth_address', 'user', 'key',
'project_domain_id', 'project_domain_name',
'user_domain_id', 'user_domain_name'):
reference[param] = CONFIG.get(ref, param)
try:
reference['auth_version'] = CONFIG.get(ref, 'auth_version')
except configparser.NoOptionError:
if self.backend_group:
av = getattr(
self.conf,
self.backend_group).swift_store_auth_version
else:
av = self.conf.glance_store.swift_store_auth_version
reference['auth_version'] = av
account_params[ref] = reference
except (ValueError, SyntaxError, configparser.NoOptionError):
LOG.exception(_LE("Invalid format of swift store config cfg"))
return account_params
def get_connection(self, conffile, rados_id):
client = rados.Rados(conffile=conffile, rados_id=rados_id)
try:
client.connect(timeout=self.connect_timeout)
except (rados.Error, rados.ObjectNotFound) as e:
if self.backend_group and len(self.conf.enabled_backends) > 1:
reason = _("Error in store configuration: %s") % e
LOG.debug(reason)
raise exceptions.BadStoreConfiguration(
store_name=self.backend_group, reason=reason)
else:
msg = _LE("Error connecting to ceph cluster.")
LOG.exception(msg)
raise exceptions.BackendException()
try:
yield client
finally:
client.shutdown()
def add(self,
image_id,
image_file,
image_size,
hashing_algo,
context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param hashing_algo: A hashlib algorithm identifier (string)
:param context: A context object
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
exists
"""
os_hash_value = utils.get_hasher(hashing_algo, False)
checksum = utils.get_hasher('md5', False)
image_name = str(image_id)
with self.get_connection(conffile=self.conf_file,
rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
# Librados's get_fsid is represented as binary
# in py3 instead of str as it is in py2.
# This is causing problems with ceph.
# Decode binary to str fixes these issues.
# Fix with encodeutils.safe_decode CAN BE REMOVED
# after librados's fix will be stable.
#
# More informations:
# https://bugs.launchpad.net/glance-store/+bug/1816721
# https://bugs.launchpad.net/cinder/+bug/1816468
# https://tracker.ceph.com/issues/38381
fsid = encodeutils.safe_decode(conn.get_fsid())
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.WRITE_CHUNKSIZE, 2))
LOG.debug('creating image %s with order %d and size %d',
image_name, order, image_size)
if image_size == 0:
LOG.warning(
_LW("Since image size is zero we will be "
"doing resize-before-write which will be "
"slower than normal"))
try:
loc = self._create_image(fsid, conn, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
msg = _('RBD image %s already exists') % image_id
raise exceptions.Duplicate(message=msg)
try:
with rbd.Image(ioctx, image_name) as image:
bytes_written = 0
offset = 0
chunks = utils.chunkreadable(image_file,
self.WRITE_CHUNKSIZE)
for chunk in chunks:
# NOTE(jokke): If we don't know image size we need
# to resize it on write. The resize amount will
# ramp up to 8 gigs.
chunk_length = len(chunk)
self.size = self._resize_on_write(
image, image_size, bytes_written, chunk_length)
bytes_written += chunk_length
if not (self.thin_provisioning and not any(chunk)):
image.write(chunk, offset)
offset += chunk_length
os_hash_value.update(chunk)
checksum.update(chunk)
if verifier:
verifier.update(chunk)
# Lets trim the image in case we overshoot with resize
if image_size == 0:
image.resize(bytes_written)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except rbd.NoSpace:
log_msg = (_LE("Failed to store image %(img_name)s "
"insufficient space available") % {
'img_name': image_name
})
LOG.error(log_msg)
# Delete image if one was created
try:
target_pool = loc.pool or self.pool
self._delete_image(target_pool, loc.image,
loc.snapshot)
except exceptions.NotFound:
pass
raise exceptions.StorageFull(message=log_msg)
except Exception as exc:
log_msg = (_LE("Failed to store image %(img_name)s "
"Store Exception %(store_exc)s") % {
'img_name': image_name,
'store_exc': exc
})
LOG.error(log_msg)
# Delete image if one was created
try:
target_pool = loc.pool or self.pool
self._delete_image(target_pool, loc.image,
loc.snapshot)
except exceptions.NotFound:
pass
raise exc
# Make sure we send back the image size whether provided or inferred.
if image_size == 0:
image_size = bytes_written
# Add store backend information to location metadata
metadata = {}
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
return (loc.get_uri(), image_size, checksum.hexdigest(),
os_hash_value.hexdigest(), metadata)
def add(self, image_id, image_file, image_size, context=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exceptions.Duplicate` if the image already
existed
`glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
ds = self.select_datastore(image_size)
if image_size > 0:
headers = {'Content-Length': image_size}
image_file = _Reader(image_file)
else:
# NOTE (arnaud): use chunk encoding when the image is still being
# generated by the server (ex: stream optimized disks generated by
# Nova).
headers = {'Transfer-Encoding': 'chunked'}
image_file = _ChunkReader(image_file)
loc = StoreLocation(
{
'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': ds.datacenter.path,
'datastore_name': ds.name,
'image_id': image_id
}, self.conf)
# NOTE(arnaud): use a decorator when the config is not tied to self
cookie = self._build_vim_cookie_header(True)
headers = dict(headers)
headers['Cookie'] = cookie
conn_class = self._get_http_conn_class()
conn = conn_class(loc.server_host)
url = urllib.parse.quote('%s?%s' % (loc.path, loc.query))
try:
conn.request('PUT', url, image_file, headers)
except IOError as e:
# When a session is not authenticated, the socket is closed by
# the server after sending the response. http_client has an open
# issue with https that raises Broken Pipe
# error instead of returning the response.
# See http://bugs.python.org/issue16062. Here, we log the error
# and continue to look into the response.
msg = _LE('Communication error sending http %(method)s request '
'to the url %(url)s.\n'
'Got IOError %(e)s') % {
'method': 'PUT',
'url': url,
'e': e
}
LOG.error(msg)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_LE('Failed to upload content of image '
'%(image)s'), {'image': image_id})
res = conn.getresponse()
if res.status == http_client.CONFLICT:
raise exceptions.Duplicate(
_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
if res.status not in (http_client.CREATED, http_client.OK):
msg = (_LE('Failed to upload content of image %(image)s. '
'The request returned an unexpected status: %(status)s.'
'\nThe response body:\n%(body)s') % {
'image': image_id,
'status': res.status,
'body': getattr(res, 'body', None)
})
LOG.error(msg)
raise exceptions.BackendException(msg)
return (loc.get_uri(), image_file.size,
image_file.checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size, context=None,
verifier=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param verifier: An object used to verify signatures for images
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises: `glance.common.exceptions.Duplicate` if the image already
existed
`glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
ds = self.select_datastore(image_size)
image_file = _Reader(image_file, verifier)
headers = {}
if image_size > 0:
headers.update({'Content-Length': image_size})
data = image_file
else:
data = utils.chunkiter(image_file, CHUNKSIZE)
loc = StoreLocation({'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': ds.datacenter.path,
'datastore_name': ds.name,
'image_id': image_id}, self.conf)
# NOTE(arnaud): use a decorator when the config is not tied to self
cookie = self._build_vim_cookie_header(True)
headers = dict(headers)
headers.update({'Cookie': cookie})
session = new_session(self.api_insecure, self.ca_file)
url = loc.https_url
try:
response = session.put(url, data=data, headers=headers)
except IOError as e:
# TODO(sigmavirus24): Figure out what the new exception type would
# be in requests.
# When a session is not authenticated, the socket is closed by
# the server after sending the response. http_client has an open
# issue with https that raises Broken Pipe
# error instead of returning the response.
# See http://bugs.python.org/issue16062. Here, we log the error
# and continue to look into the response.
msg = _LE('Communication error sending http %(method)s request '
'to the url %(url)s.\n'
'Got IOError %(e)s') % {'method': 'PUT',
'url': url,
'e': e}
LOG.error(msg)
raise exceptions.BackendException(msg)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to upload content of image '
'%(image)s'), {'image': image_id})
res = response.raw
if res.status == requests.codes.conflict:
raise exceptions.Duplicate(_("Image file %(image_id)s already "
"exists!") %
{'image_id': image_id})
if res.status not in (requests.codes.created, requests.codes.ok):
msg = (_LE('Failed to upload content of image %(image)s. '
'The request returned an unexpected status: %(status)s.'
'\nThe response body:\n%(body)s') %
{'image': image_id,
'status': res.status,
'body': getattr(res, 'body', None)})
LOG.error(msg)
raise exceptions.BackendException(msg)
return (loc.get_uri(), image_file.size,
image_file.checksum.hexdigest(), {})
def _open_cinder_volume(self, client, volume, mode):
attach_mode = 'rw' if mode == 'wb' else 'ro'
device = None
root_helper = get_root_helper(backend=self.backend_group)
priv_context.init(root_helper=shlex.split(root_helper))
host = socket.gethostname()
if self.backend_group:
use_multipath = getattr(
self.conf, self.backend_group).cinder_use_multipath
enforce_multipath = getattr(
self.conf, self.backend_group).cinder_enforce_multipath
else:
use_multipath = self.conf.glance_store.cinder_use_multipath
enforce_multipath = self.conf.glance_store.cinder_enforce_multipath
properties = connector.get_connector_properties(
root_helper, host, use_multipath, enforce_multipath)
try:
volume.reserve(volume)
except cinder_exception.ClientException as e:
msg = (_('Failed to reserve volume %(volume_id)s: %(error)s')
% {'volume_id': volume.id, 'error': e})
LOG.error(msg)
raise exceptions.BackendException(msg)
try:
connection_info = volume.initialize_connection(volume, properties)
conn = connector.InitiatorConnector.factory(
connection_info['driver_volume_type'], root_helper,
conn=connection_info)
device = conn.connect_volume(connection_info['data'])
volume.attach(None, 'glance_store', attach_mode, host_name=host)
volume = self._wait_volume_status(volume, 'attaching', 'in-use')
if (connection_info['driver_volume_type'] == 'rbd' and
not conn.do_local_attach):
yield device['path']
else:
with temporary_chown(device['path'],
backend=self.backend_group), \
open(device['path'], mode) as f:
yield f
except Exception:
LOG.exception(_LE('Exception while accessing to cinder volume '
'%(volume_id)s.'), {'volume_id': volume.id})
raise
finally:
if volume.status == 'in-use':
volume.begin_detaching(volume)
elif volume.status == 'attaching':
volume.unreserve(volume)
if device:
try:
conn.disconnect_volume(connection_info['data'], device)
except Exception:
LOG.exception(_LE('Failed to disconnect volume '
'%(volume_id)s.'),
{'volume_id': volume.id})
try:
volume.terminate_connection(volume, properties)
except Exception:
LOG.exception(_LE('Failed to terminate connection of volume '
'%(volume_id)s.'), {'volume_id': volume.id})
try:
client.volumes.detach(volume)
except Exception:
LOG.exception(_LE('Failed to detach volume %(volume_id)s.'),
{'volume_id': volume.id})
def mount(self, fstype, export, vol_name, mountpoint, host,
rootwrap_helper, options):
"""Ensure a mountpoint is available for an attachment, mounting it
if necessary.
If this is the first attachment on this mountpoint, we will mount it
with:
mount -t <fstype> <options> <export> <mountpoint>
:param fstype: The filesystem type to be passed to mount command.
:param export: The type-specific identifier of the filesystem to be
mounted. e.g. for nfs 'host.example.com:/mountpoint'.
:param vol_name: The name of the volume on the remote filesystem.
:param mountpoint: The directory where the filesystem will be
mounted on the local compute host.
:param host: The host the volume will be attached to.
:param options: An arbitrary list of additional arguments to be
passed to the mount command immediate before export
and mountpoint.
"""
LOG.debug('_HostMountState.mount(fstype=%(fstype)s, '
'export=%(export)s, vol_name=%(vol_name)s, %(mountpoint)s, '
'options=%(options)s)',
{'fstype': fstype, 'export': export, 'vol_name': vol_name,
'mountpoint': mountpoint, 'options': options})
with self._get_locked(mountpoint) as mount:
if not os.path.ismount(mountpoint):
LOG.debug('Mounting %(mountpoint)s',
{'mountpoint': mountpoint})
os.makedirs(mountpoint)
mount_cmd = ['mount', '-t', fstype]
if options is not None:
mount_cmd.extend(options)
mount_cmd.extend([export, mountpoint])
try:
processutils.execute(*mount_cmd, run_as_root=True,
root_helper=rootwrap_helper)
except Exception:
# Check to see if mountpoint is mounted despite the error
# eg it was already mounted
if os.path.ismount(mountpoint):
# We're not going to raise the exception because we're
# in the desired state anyway. However, this is still
# unusual so we'll log it.
LOG.exception(_LE('Error mounting %(fstype)s export '
'%(export)s on %(mountpoint)s. '
'Continuing because mountpount is '
'mounted despite this.'),
{'fstype': fstype, 'export': export,
'mountpoint': mountpoint})
else:
# If the mount failed there's no reason for us to keep
# a record of it. It will be created again if the
# caller retries.
# Delete while holding lock
del self.mountpoints[mountpoint]
raise
mount.add_attachment(vol_name, host)
LOG.debug('_HostMountState.mount() for %(mountpoint)s '
'completed successfully',
{'mountpoint': mountpoint})
def add(self,
image_id,
image_file,
image_size,
hashing_algo,
context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param hashing_algo: A hashlib algorithm identifier (string)
:param context: A context object
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
exists
"""
checksum = hashlib.md5()
os_hash_value = hashlib.new(str(hashing_algo))
image_name = str(image_id)
with self.get_connection(conffile=self.conf_file,
rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.WRITE_CHUNKSIZE, 2))
LOG.debug('creating image %s with order %d and size %d',
image_name, order, image_size)
if image_size == 0:
LOG.warning(
_("since image size is zero we will be doing "
"resize-before-write for each chunk which "
"will be considerably slower than normal"))
try:
loc = self._create_image(fsid, conn, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
msg = _('RBD image %s already exists') % image_id
raise exceptions.Duplicate(message=msg)
try:
with rbd.Image(ioctx, image_name) as image:
bytes_written = 0
offset = 0
chunks = utils.chunkreadable(image_file,
self.WRITE_CHUNKSIZE)
for chunk in chunks:
# If the image size provided is zero we need to do
# a resize for the amount we are writing. This will
# be slower so setting a higher chunk size may
# speed things up a bit.
if image_size == 0:
chunk_length = len(chunk)
length = offset + chunk_length
bytes_written += chunk_length
LOG.debug(
_("resizing image to %s KiB") %
(length / units.Ki))
image.resize(length)
LOG.debug(
_("writing chunk at offset %s") % (offset))
offset += image.write(chunk, offset)
os_hash_value.update(chunk)
checksum.update(chunk)
if verifier:
verifier.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except Exception as exc:
log_msg = (_LE("Failed to store image %(img_name)s "
"Store Exception %(store_exc)s") % {
'img_name': image_name,
'store_exc': exc
})
LOG.error(log_msg)
# Delete image if one was created
try:
target_pool = loc.pool or self.pool
self._delete_image(target_pool, loc.image,
loc.snapshot)
except exceptions.NotFound:
pass
raise exc
# Make sure we send back the image size whether provided or inferred.
if image_size == 0:
image_size = bytes_written
# Add store backend information to location metadata
metadata = {}
if self.backend_group:
metadata['backend'] = u"%s" % self.backend_group
return (loc.get_uri(), image_size, checksum.hexdigest(),
os_hash_value.hexdigest(), metadata)
def rewind(self):
try:
self.data.seek(0)
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to rewind image content'))
def rewind(self):
try:
self.data.seek(0)
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to rewind image content"))
def add(self, image_id, image_file, image_size, context=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exceptions.Duplicate` if the image already
existed
`glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
if image_size > 0:
headers = {'Content-Length': image_size}
image_file = _Reader(image_file)
else:
# NOTE (arnaud): use chunk encoding when the image is still being
# generated by the server (ex: stream optimized disks generated by
# Nova).
headers = {'Transfer-Encoding': 'chunked'}
image_file = _ChunkReader(image_file)
loc = StoreLocation({'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id}, self.conf)
# NOTE(arnaud): use a decorator when the config is not tied to self
for i in range(self.api_retry_count + 1):
cookie = self._build_vim_cookie_header(
self.session.vim.client.options.transport.cookiejar)
headers = dict(headers.items() + {'Cookie': cookie}.items())
try:
conn = self._get_http_conn('PUT', loc, headers,
content=image_file)
res = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to upload content of image '
'%(image)s'), {'image': image_id})
if res.status == httplib.UNAUTHORIZED:
self.reset_session(force=True)
image_file.rewind()
continue
if res.status == httplib.CONFLICT:
raise exceptions.Duplicate(_("Image file %(image_id)s already "
"exists!") %
{'image_id': image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = (_LE('Failed to upload content of image %(image)s') %
{'image': image_id})
LOG.error(msg)
raise exceptions.UnexpectedStatus(status=res.status,
body=res.read())
break
return (loc.get_uri(), image_file.size,
image_file.checksum.hexdigest(), {})
def configure_add(self):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
if self.backend_group:
store_conf = getattr(self.conf, self.backend_group)
else:
store_conf = self.conf.glance_store
fdir = store_conf.filesystem_store_datadir
fdirs = store_conf.filesystem_store_datadirs
fstore_perm = store_conf.filesystem_store_file_perm
meta_file = store_conf.filesystem_store_metadata_file
self.chunk_size = store_conf.filesystem_store_chunk_size
self.READ_CHUNKSIZE = self.chunk_size
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
if not (fdir or fdirs):
reason = (_("Specify at least 'filesystem_store_datadir' or "
"'filesystem_store_datadirs' option"))
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="filesystem",
reason=reason)
if fdir and fdirs:
reason = (_("Specify either 'filesystem_store_datadir' or "
"'filesystem_store_datadirs' option"))
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="filesystem",
reason=reason)
if fstore_perm > 0:
perm = int(str(fstore_perm), 8)
if not perm & stat.S_IRUSR:
reason = _LE("Specified an invalid "
"'filesystem_store_file_perm' option which "
"could make image file to be unaccessible by "
"glance service.")
LOG.error(reason)
reason = _("Invalid 'filesystem_store_file_perm' option.")
raise exceptions.BadStoreConfiguration(store_name="filesystem",
reason=reason)
self.multiple_datadirs = False
directory_paths = set()
if fdir:
self.datadir = fdir
directory_paths.add(self.datadir)
else:
self.multiple_datadirs = True
self.priority_data_map = {}
for datadir in fdirs:
(datadir_path,
priority) = self._get_datadir_path_and_priority(datadir)
priority_paths = self.priority_data_map.setdefault(
int(priority), [])
self._check_directory_paths(datadir_path, directory_paths,
priority_paths)
directory_paths.add(datadir_path)
priority_paths.append(datadir_path)
self.priority_list = sorted(self.priority_data_map,
reverse=True)
self._create_image_directories(directory_paths)
if meta_file:
self._validate_metadata(meta_file)
def add(self,
image_id,
image_file,
image_size,
context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param context: The request context
:param verifier: An object used to verify signatures for images
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance_store.exceptions.Duplicate` if the image already
existed
"""
self._check_context(context, require_tenant=True)
client = get_cinderclient(self.conf, context)
checksum = hashlib.md5()
bytes_written = 0
size_gb = int((image_size + units.Gi - 1) / units.Gi)
if size_gb == 0:
size_gb = 1
name = "image-%s" % image_id
owner = context.tenant
metadata = {
'glance_image_id': image_id,
'image_size': str(image_size),
'image_owner': owner
}
LOG.debug('Creating a new volume: image_size=%d size_gb=%d',
image_size, size_gb)
if image_size == 0:
LOG.info(
_LI("Since image size is zero, we will be doing "
"resize-before-write for each GB which "
"will be considerably slower than normal."))
volume = client.volumes.create(size_gb, name=name, metadata=metadata)
volume = self._wait_volume_status(volume, 'creating', 'available')
failed = True
need_extend = True
buf = None
try:
while need_extend:
with self._open_cinder_volume(client, volume, 'wb') as f:
f.seek(bytes_written)
if buf:
f.write(buf)
bytes_written += len(buf)
while True:
buf = image_file.read(self.WRITE_CHUNKSIZE)
if not buf:
need_extend = False
break
checksum.update(buf)
if verifier:
verifier.update(buf)
if (bytes_written + len(buf) > size_gb * units.Gi
and image_size == 0):
break
f.write(buf)
bytes_written += len(buf)
if need_extend:
size_gb += 1
LOG.debug("Extending volume %(volume_id)s to %(size)s GB.",
{
'volume_id': volume.id,
'size': size_gb
})
volume.extend(volume, size_gb)
try:
volume = self._wait_volume_status(
volume, 'extending', 'available')
except exceptions.BackendException:
raise exceptions.StorageFull()
failed = False
except IOError as e:
# Convert IOError reasons to Glance Store exceptions
errors = {
errno.EFBIG: exceptions.StorageFull(),
errno.ENOSPC: exceptions.StorageFull(),
errno.EACCES: exceptions.StorageWriteDenied()
}
raise errors.get(e.errno, e)
finally:
if failed:
LOG.error(_LE("Failed to write to volume %(volume_id)s."),
{'volume_id': volume.id})
try:
volume.delete()
except Exception:
LOG.exception(
_LE('Failed to delete of volume '
'%(volume_id)s.'), {'volume_id': volume.id})
if image_size == 0:
metadata.update({'image_size': str(bytes_written)})
volume.update_all_metadata(metadata)
volume.update_readonly_flag(volume, True)
checksum_hex = checksum.hexdigest()
LOG.debug(
"Wrote %(bytes_written)d bytes to volume %(volume_id)s "
"with checksum %(checksum_hex)s.", {
'bytes_written': bytes_written,
'volume_id': volume.id,
'checksum_hex': checksum_hex
})
return ('cinder://%s' % volume.id, bytes_written, checksum_hex, {})
def add(self, image_id, image_file, image_size, context=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exceptions.Duplicate` if the image already
existed
`glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
loc = StoreLocation({
'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id
})
# NOTE(arnaud): use a decorator when the config is not tied to self
for i in range(self.api_retry_count + 1):
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {
'Connection': 'Keep-Alive',
'Cookie': cookie,
'Transfer-Encoding': 'chunked'
}
try:
conn = self._get_http_conn('PUT',
loc,
headers,
content=image_file)
res = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_LE('Failed to upload content of image '
'%(image)s'), {'image': image_id})
if res.status == httplib.UNAUTHORIZED:
self._create_session()
image_file.rewind()
continue
if res.status == httplib.CONFLICT:
raise exceptions.Duplicate(
_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = (_LE('Failed to upload content of image %(image)s') % {
'image': image_id
})
LOG.error(msg)
raise exceptions.UnexpectedStatus(status=res.status,
body=res.read())
break
return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def configure_add(self):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
if not (self.conf.glance_store.filesystem_store_datadir or
self.conf.glance_store.filesystem_store_datadirs):
reason = (_("Specify at least 'filesystem_store_datadir' or "
"'filesystem_store_datadirs' option"))
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="filesystem",
reason=reason)
if (self.conf.glance_store.filesystem_store_datadir and
self.conf.glance_store.filesystem_store_datadirs):
reason = (_("Specify either 'filesystem_store_datadir' or "
"'filesystem_store_datadirs' option"))
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="filesystem",
reason=reason)
if self.conf.glance_store.filesystem_store_file_perm > 0:
perm = int(str(self.conf.glance_store.filesystem_store_file_perm),
8)
if not perm & stat.S_IRUSR:
reason = _LE("Specified an invalid "
"'filesystem_store_file_perm' option which "
"could make image file to be unaccessible by "
"glance service.")
LOG.error(reason)
reason = _("Invalid 'filesystem_store_file_perm' option.")
raise exceptions.BadStoreConfiguration(store_name="filesystem",
reason=reason)
self.multiple_datadirs = False
directory_paths = set()
if self.conf.glance_store.filesystem_store_datadir:
self.datadir = self.conf.glance_store.filesystem_store_datadir
directory_paths.add(self.datadir)
else:
self.multiple_datadirs = True
self.priority_data_map = {}
for datadir in self.conf.glance_store.filesystem_store_datadirs:
(datadir_path,
priority) = self._get_datadir_path_and_priority(datadir)
priority_paths = self.priority_data_map.setdefault(
int(priority), [])
self._check_directory_paths(datadir_path, directory_paths,
priority_paths)
directory_paths.add(datadir_path)
priority_paths.append(datadir_path)
self.priority_list = sorted(self.priority_data_map,
reverse=True)
self._create_image_directories(directory_paths)
metadata_file = self.conf.glance_store.filesystem_store_metadata_file
if metadata_file:
self._validate_metadata(metadata_file)
def add(self, image_id, image_file, image_size, hashing_algo, context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param hashing_algo: A hashlib algorithm identifier (string)
:param context: A context object
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
exists
"""
checksum = hashlib.md5()
os_hash_value = hashlib.new(str(hashing_algo))
image_name = str(image_id)
with self.get_connection(conffile=self.conf_file,
rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
# Librados's get_fsid is represented as binary
# in py3 instead of str as it is in py2.
# This is causing problems with ceph.
# Decode binary to str fixes these issues.
# Fix with encodeutils.safe_decode CAN BE REMOVED
# after librados's fix will be stable.
#
# More informations:
# https://bugs.launchpad.net/glance-store/+bug/1816721
# https://bugs.launchpad.net/cinder/+bug/1816468
# https://tracker.ceph.com/issues/38381
fsid = encodeutils.safe_decode(conn.get_fsid())
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.WRITE_CHUNKSIZE, 2))
LOG.debug('creating image %s with order %d and size %d',
image_name, order, image_size)
if image_size == 0:
LOG.warning(_("since image size is zero we will be doing "
"resize-before-write for each chunk which "
"will be considerably slower than normal"))
try:
loc = self._create_image(fsid, conn, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
msg = _('RBD image %s already exists') % image_id
raise exceptions.Duplicate(message=msg)
try:
with rbd.Image(ioctx, image_name) as image:
bytes_written = 0
offset = 0
chunks = utils.chunkreadable(image_file,
self.WRITE_CHUNKSIZE)
for chunk in chunks:
# If the image size provided is zero we need to do
# a resize for the amount we are writing. This will
# be slower so setting a higher chunk size may
# speed things up a bit.
if image_size == 0:
chunk_length = len(chunk)
length = offset + chunk_length
bytes_written += chunk_length
LOG.debug(_("resizing image to %s KiB") %
(length / units.Ki))
image.resize(length)
LOG.debug(_("writing chunk at offset %s") %
(offset))
offset += image.write(chunk, offset)
os_hash_value.update(chunk)
checksum.update(chunk)
if verifier:
verifier.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except rbd.NoSpace:
log_msg = (_LE("Failed to store image %(img_name)s "
"insufficient space available") %
{'img_name': image_name})
LOG.error(log_msg)
# Delete image if one was created
try:
target_pool = loc.pool or self.pool
self._delete_image(target_pool, loc.image,
loc.snapshot)
except exceptions.NotFound:
pass
raise exceptions.StorageFull(message=log_msg)
except Exception as exc:
log_msg = (_LE("Failed to store image %(img_name)s "
"Store Exception %(store_exc)s") %
{'img_name': image_name,
'store_exc': exc})
LOG.error(log_msg)
# Delete image if one was created
try:
target_pool = loc.pool or self.pool
self._delete_image(target_pool, loc.image,
loc.snapshot)
except exceptions.NotFound:
pass
raise exc
# Make sure we send back the image size whether provided or inferred.
if image_size == 0:
image_size = bytes_written
# Add store backend information to location metadata
metadata = {}
if self.backend_group:
metadata['backend'] = u"%s" % self.backend_group
return (loc.get_uri(),
image_size,
checksum.hexdigest(),
os_hash_value.hexdigest(),
metadata)
def _open_cinder_volume(self, client, volume, mode):
attach_mode = 'rw' if mode == 'wb' else 'ro'
device = None
root_helper = self.get_root_helper()
priv_context.init(root_helper=shlex.split(root_helper))
host = socket.gethostname()
use_multipath = self.store_conf.cinder_use_multipath
enforce_multipath = self.store_conf.cinder_enforce_multipath
mount_point_base = self.store_conf.cinder_mount_point_base
properties = connector.get_connector_properties(
root_helper, host, use_multipath, enforce_multipath)
try:
volume.reserve(volume)
except cinder_exception.ClientException as e:
msg = (_('Failed to reserve volume %(volume_id)s: %(error)s') % {
'volume_id': volume.id,
'error': e
})
LOG.error(msg)
raise exceptions.BackendException(msg)
try:
connection_info = volume.initialize_connection(volume, properties)
conn = connector.InitiatorConnector.factory(
connection_info['driver_volume_type'],
root_helper,
conn=connection_info)
if connection_info['driver_volume_type'] == 'nfs':
if volume.encrypted:
volume.unreserve(volume)
volume.delete()
msg = (_('Encrypted volume creation for cinder nfs is not '
'supported from glance_store. Failed to create '
'volume %(volume_id)s') % {
'volume_id': volume.id
})
LOG.error(msg)
raise exceptions.BackendException(msg)
@utils.synchronized(connection_info['data']['export'])
def connect_volume_nfs():
data = connection_info['data']
export = data['export']
vol_name = data['name']
mountpoint = self._get_mount_path(
export, os.path.join(mount_point_base, 'nfs'))
options = data['options']
self.mount.mount('nfs', export, vol_name, mountpoint, host,
root_helper, options)
return {'path': os.path.join(mountpoint, vol_name)}
device = connect_volume_nfs()
else:
device = conn.connect_volume(connection_info['data'])
volume.attach(None, 'glance_store', attach_mode, host_name=host)
volume = self._wait_volume_status(volume, 'attaching', 'in-use')
if (connection_info['driver_volume_type'] == 'rbd'
and not conn.do_local_attach):
yield device['path']
else:
with self.temporary_chown(device['path']), open(
device['path'], mode) as f:
yield f
except Exception:
LOG.exception(
_LE('Exception while accessing to cinder volume '
'%(volume_id)s.'), {'volume_id': volume.id})
raise
finally:
if volume.status == 'in-use':
volume.begin_detaching(volume)
elif volume.status == 'attaching':
volume.unreserve(volume)
if device:
try:
if connection_info['driver_volume_type'] == 'nfs':
@utils.synchronized(connection_info['data']['export'])
def disconnect_volume_nfs():
path, vol_name = device['path'].rsplit('/', 1)
self.mount.umount(vol_name, path, host,
root_helper)
disconnect_volume_nfs()
else:
conn.disconnect_volume(connection_info['data'], device)
except Exception:
LOG.exception(
_LE('Failed to disconnect volume '
'%(volume_id)s.'), {'volume_id': volume.id})
try:
volume.terminate_connection(volume, properties)
except Exception:
LOG.exception(
_LE('Failed to terminate connection of volume '
'%(volume_id)s.'), {'volume_id': volume.id})
try:
client.volumes.detach(volume)
except Exception:
LOG.exception(_LE('Failed to detach volume %(volume_id)s.'),
{'volume_id': volume.id})
def add(self, image_id, image_file, image_size, context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param context: The request context
:param verifier: An object used to verify signatures for images
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance_store.exceptions.Duplicate` if the image already
existed
"""
self._check_context(context, require_tenant=True)
client = get_cinderclient(self.conf, context)
checksum = hashlib.md5()
bytes_written = 0
size_gb = int(math.ceil(float(image_size) / units.Gi))
if size_gb == 0:
size_gb = 1
name = "image-%s" % image_id
owner = context.tenant
metadata = {'glance_image_id': image_id,
'image_size': str(image_size),
'image_owner': owner}
LOG.debug('Creating a new volume: image_size=%d size_gb=%d',
image_size, size_gb)
if image_size == 0:
LOG.info(_LI("Since image size is zero, we will be doing "
"resize-before-write for each GB which "
"will be considerably slower than normal."))
volume = client.volumes.create(size_gb, name=name, metadata=metadata)
volume = self._wait_volume_status(volume, 'creating', 'available')
failed = True
need_extend = True
buf = None
try:
while need_extend:
with self._open_cinder_volume(client, volume, 'wb') as f:
f.seek(bytes_written)
if buf:
f.write(buf)
bytes_written += len(buf)
while True:
buf = image_file.read(self.WRITE_CHUNKSIZE)
if not buf:
need_extend = False
break
checksum.update(buf)
if verifier:
verifier.update(buf)
if (bytes_written + len(buf) > size_gb * units.Gi and
image_size == 0):
break
f.write(buf)
bytes_written += len(buf)
if need_extend:
size_gb += 1
LOG.debug("Extending volume %(volume_id)s to %(size)s GB.",
{'volume_id': volume.id, 'size': size_gb})
volume.extend(volume, size_gb)
try:
volume = self._wait_volume_status(volume,
'extending',
'available')
except exceptions.BackendException:
raise exceptions.StorageFull()
failed = False
except IOError as e:
# Convert IOError reasons to Glance Store exceptions
errors = {errno.EFBIG: exceptions.StorageFull(),
errno.ENOSPC: exceptions.StorageFull(),
errno.EACCES: exceptions.StorageWriteDenied()}
raise errors.get(e.errno, e)
finally:
if failed:
LOG.error(_LE("Failed to write to volume %(volume_id)s."),
{'volume_id': volume.id})
try:
volume.delete()
except Exception:
LOG.exception(_LE('Failed to delete of volume '
'%(volume_id)s.'),
{'volume_id': volume.id})
if image_size == 0:
metadata.update({'image_size': str(bytes_written)})
volume.update_all_metadata(metadata)
volume.update_readonly_flag(volume, True)
checksum_hex = checksum.hexdigest()
LOG.debug("Wrote %(bytes_written)d bytes to volume %(volume_id)s "
"with checksum %(checksum_hex)s.",
{'bytes_written': bytes_written,
'volume_id': volume.id,
'checksum_hex': checksum_hex})
return ('cinder://%s' % volume.id, bytes_written, checksum_hex, {})
def add(self,
image_id,
image_file,
image_size,
hashing_algo,
context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param hashing_algo: A hashlib algorithm identifier (string)
:param context: The request context
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
exists
"""
self._check_context(context, require_tenant=True)
client = self.get_cinderclient(context)
os_hash_value = utils.get_hasher(hashing_algo, False)
checksum = utils.get_hasher('md5', False)
bytes_written = 0
size_gb = int(math.ceil(float(image_size) / units.Gi))
if size_gb == 0:
size_gb = 1
name = "image-%s" % image_id
owner = context.project_id
metadata = {
'glance_image_id': image_id,
'image_size': str(image_size),
'image_owner': owner
}
volume_type = self.store_conf.cinder_volume_type
LOG.debug('Creating a new volume: image_size=%d size_gb=%d type=%s',
image_size, size_gb, volume_type or 'None')
if image_size == 0:
LOG.info(
_LI("Since image size is zero, we will be doing "
"resize-before-write for each GB which "
"will be considerably slower than normal."))
volume = client.volumes.create(size_gb,
name=name,
metadata=metadata,
volume_type=volume_type)
volume = self._wait_volume_status(volume, 'creating', 'available')
size_gb = volume.size
failed = True
need_extend = True
buf = None
try:
while need_extend:
with self._open_cinder_volume(client, volume, 'wb') as f:
f.seek(bytes_written)
if buf:
f.write(buf)
bytes_written += len(buf)
while True:
buf = image_file.read(self.WRITE_CHUNKSIZE)
if not buf:
need_extend = False
break
os_hash_value.update(buf)
checksum.update(buf)
if verifier:
verifier.update(buf)
if (bytes_written + len(buf) > size_gb * units.Gi
and image_size == 0):
break
f.write(buf)
bytes_written += len(buf)
if need_extend:
size_gb += 1
LOG.debug("Extending volume %(volume_id)s to %(size)s GB.",
{
'volume_id': volume.id,
'size': size_gb
})
volume.extend(volume, size_gb)
try:
volume = self._wait_volume_status(
volume, 'extending', 'available')
size_gb = volume.size
except exceptions.BackendException:
raise exceptions.StorageFull()
failed = False
except IOError as e:
# Convert IOError reasons to Glance Store exceptions
errors = {
errno.EFBIG: exceptions.StorageFull(),
errno.ENOSPC: exceptions.StorageFull(),
errno.EACCES: exceptions.StorageWriteDenied()
}
raise errors.get(e.errno, e)
finally:
if failed:
LOG.error(_LE("Failed to write to volume %(volume_id)s."),
{'volume_id': volume.id})
try:
volume.delete()
except Exception:
LOG.exception(
_LE('Failed to delete of volume '
'%(volume_id)s.'), {'volume_id': volume.id})
if image_size == 0:
metadata.update({'image_size': str(bytes_written)})
volume.update_all_metadata(metadata)
volume.update_readonly_flag(volume, True)
hash_hex = os_hash_value.hexdigest()
checksum_hex = checksum.hexdigest()
LOG.debug(
"Wrote %(bytes_written)d bytes to volume %(volume_id)s "
"with checksum %(checksum_hex)s.", {
'bytes_written': bytes_written,
'volume_id': volume.id,
'checksum_hex': checksum_hex
})
image_metadata = {}
location_url = 'cinder://%s' % volume.id
if self.backend_group:
image_metadata['store'] = u"%s" % self.backend_group
location_url = 'cinder://%s/%s' % (self.backend_group, volume.id)
return (location_url, bytes_written, checksum_hex, hash_hex,
image_metadata)
def add(self,
image_id,
image_file,
image_size,
hashing_algo,
context=None,
verifier=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param hashing_algo: A hashlib algorithm identifier (string)
:param context: A context object
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
exists
:raises: `glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
ds = self.select_datastore(image_size)
image_file = _Reader(image_file, hashing_algo, verifier)
headers = {}
if image_size > 0:
headers.update({'Content-Length': six.text_type(image_size)})
data = image_file
else:
data = utils.chunkiter(image_file, CHUNKSIZE)
loc = StoreLocation(
{
'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': ds.datacenter.path,
'datastore_name': ds.name,
'image_id': image_id
},
self.conf,
backend_group=self.backend_group)
# NOTE(arnaud): use a decorator when the config is not tied to self
cookie = self._build_vim_cookie_header(True)
headers = dict(headers)
headers.update({'Cookie': cookie})
session = new_session(self.api_insecure, self.ca_file)
url = loc.https_url
try:
response = session.put(url, data=data, headers=headers)
except IOError as e:
# TODO(sigmavirus24): Figure out what the new exception type would
# be in requests.
# When a session is not authenticated, the socket is closed by
# the server after sending the response. http_client has an open
# issue with https that raises Broken Pipe
# error instead of returning the response.
# See http://bugs.python.org/issue16062. Here, we log the error
# and continue to look into the response.
msg = _LE('Communication error sending http %(method)s request '
'to the url %(url)s.\n'
'Got IOError %(e)s') % {
'method': 'PUT',
'url': url,
'e': e
}
LOG.error(msg)
raise exceptions.BackendException(msg)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_LE('Failed to upload content of image '
'%(image)s'), {'image': image_id})
res = response.raw
if res.status == requests.codes.conflict:
raise exceptions.Duplicate(
_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
if res.status not in (requests.codes.created, requests.codes.ok):
msg = (_LE('Failed to upload content of image %(image)s. '
'The request returned an unexpected status: %(status)s.'
'\nThe response body:\n%(body)s') % {
'image': image_id,
'status': res.status,
'body': getattr(res, 'body', None)
})
LOG.error(msg)
raise exceptions.BackendException(msg)
metadata = {}
if self.backend_group:
metadata['backend'] = u"%s" % self.backend_group
return (loc.get_uri(), image_file.size,
image_file.checksum.hexdigest(),
image_file.os_hash_value.hexdigest(), metadata)
def add(self, image_id, image_file, image_size, context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param verifier: An object used to verify signatures for images
:retval: tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with self.get_connection(conffile=self.conf_file,
rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.WRITE_CHUNKSIZE, 2))
LOG.debug('creating image %s with order %d and size %d',
image_name, order, image_size)
if image_size == 0:
LOG.warning(_("since image size is zero we will be doing "
"resize-before-write for each chunk which "
"will be considerably slower than normal"))
try:
loc = self._create_image(fsid, conn, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
msg = _('RBD image %s already exists') % image_id
raise exceptions.Duplicate(message=msg)
try:
with rbd.Image(ioctx, image_name) as image:
bytes_written = 0
offset = 0
chunks = utils.chunkreadable(image_file,
self.WRITE_CHUNKSIZE)
for chunk in chunks:
# If the image size provided is zero we need to do
# a resize for the amount we are writing. This will
# be slower so setting a higher chunk size may
# speed things up a bit.
if image_size == 0:
chunk_length = len(chunk)
length = offset + chunk_length
bytes_written += chunk_length
LOG.debug(_("resizing image to %s KiB") %
(length / units.Ki))
image.resize(length)
LOG.debug(_("writing chunk at offset %s") %
(offset))
offset += image.write(chunk, offset)
checksum.update(chunk)
if verifier:
verifier.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except Exception as exc:
log_msg = (_LE("Failed to store image %(img_name)s "
"Store Exception %(store_exc)s") %
{'img_name': image_name,
'store_exc': exc})
LOG.error(log_msg)
# Delete image if one was created
try:
target_pool = loc.pool or self.pool
self._delete_image(target_pool, loc.image,
loc.snapshot)
except exceptions.NotFound:
pass
raise exc
# Make sure we send back the image size whether provided or inferred.
if image_size == 0:
image_size = bytes_written
return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size, context=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exceptions.Duplicate` if the image already
existed
`glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
ds = self.select_datastore(image_size)
if image_size > 0:
headers = {'Content-Length': image_size}
image_file = _Reader(image_file)
else:
# NOTE (arnaud): use chunk encoding when the image is still being
# generated by the server (ex: stream optimized disks generated by
# Nova).
headers = {'Transfer-Encoding': 'chunked'}
image_file = _ChunkReader(image_file)
loc = StoreLocation({'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': ds.datacenter.path,
'datastore_name': ds.name,
'image_id': image_id}, self.conf)
# NOTE(arnaud): use a decorator when the config is not tied to self
cookie = self._build_vim_cookie_header(True)
headers = dict(headers.items() + {'Cookie': cookie}.items())
conn_class = self._get_http_conn_class()
conn = conn_class(loc.server_host)
url = urlparse.quote('%s?%s' % (loc.path, loc.query))
try:
conn.request('PUT', url, image_file, headers)
except IOError as e:
# When a session is not authenticated, the socket is closed by
# the server after sending the response. httplib has an open
# issue with https that raises Broken Pipe
# error instead of returning the response.
# See http://bugs.python.org/issue16062. Here, we log the error
# and continue to look into the response.
msg = _LE('Communication error sending http %(method)s request'
'to the url %(url)s.\n'
'Got IOError %(e)s') % {'method': 'PUT',
'url': url,
'e': e}
LOG.error(msg)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to upload content of image '
'%(image)s'), {'image': image_id})
res = conn.getresponse()
if res.status == httplib.CONFLICT:
raise exceptions.Duplicate(_("Image file %(image_id)s already "
"exists!") %
{'image_id': image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = (_LE('Failed to upload content of image %(image)s. '
'The request returned an unexpected status: %(status)s.'
'\nThe response body:\n%(body)s') %
{'image': image_id,
'status': res.status,
'body': res.body})
LOG.error(msg)
raise exceptions.BackendException(msg)
return (loc.get_uri(), image_file.size,
image_file.checksum.hexdigest(), {})
