def _get_datadir_path_and_priority(self, datadir):
"""
Gets directory paths and its priority from
filesystem_store_datadirs option in glance-api.conf.
:datadir is directory path with its priority.
:returns datadir_path as directory path
priority as priority associated with datadir_path
:raise BadStoreConfiguration exception if priority is invalid or
empty directory path is specified.
"""
priority = 0
parts = map(lambda x: x.strip(), datadir.rsplit(":", 1))
datadir_path = parts[0]
if len(parts) == 2 and parts[1]:
priority = parts[1]
if not priority.isdigit():
msg = (_("Invalid priority value %(priority)s in "
"filesystem configuration") % {'priority': priority})
LOG.exception(msg)
raise exceptions.BadStoreConfiguration(
store_name="filesystem", reason=msg)
if not datadir_path:
msg = _("Invalid directory specified in filesystem configuration")
LOG.exception(msg)
raise exceptions.BadStoreConfiguration(
store_name="filesystem", reason=msg)
return datadir_path, priority
def get_size(self, location, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file and returns the image size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises `glance.store.exceptions.NotFound` if image does not exist
:rtype int
"""
loc = location.store_location
try:
self._check_context(context)
volume = get_cinderclient(self.conf,
context).volumes.get(loc.volume_id)
# GB unit convert to byte
return volume.size * (1024 ** 3)
except cinder_exception.NotFound as e:
reason = _("Failed to get image size due to "
"volume can not be found: %s") % self.volume_id
LOG.error(reason)
raise exceptions.NotFound(reason)
except Exception as e:
LOG.exception(_("Failed to get image size due to "
"internal error: %s") % e)
return 0
def _get_metadata(self):
metadata_file = self.conf.glance_store.filesystem_store_metadata_file
if metadata_file is None:
return {}
try:
with open(metadata_file, 'r') as fptr:
metadata = jsonutils.load(fptr)
glance.store.check_location_metadata(metadata)
return metadata
except exceptions.BackendException as bee:
LOG.error(_('The JSON in the metadata file %(file)s could not '
'be used: %(bee)s An empty dictionary will be '
'returned to the client.')
% dict(file=metadata_file, bee=str(bee)))
return {}
except IOError as ioe:
LOG.error(_('The path for the metadata file %(file)s could not be '
'opened: %(io)s An empty dictionary will be returned '
'to the client.')
% dict(file=metadata_file, io=ioe))
return {}
except Exception as ex:
LOG.exception(_('An error occurred processing the storage systems '
'meta data file: %s. An empty dictionary will be '
'returned to the client.') % str(ex))
return {}
def create_bucket_if_missing(bucket, s3_conn):
"""
Creates a missing bucket in S3 if the
``s3_store_create_bucket_on_put`` option is set.
:param bucket: Name of bucket to create
:param s3_conn: Connection to S3
"""
from boto.exception import S3ResponseError
try:
s3_conn.get_bucket(bucket)
except S3ResponseError as e:
if e.status == httplib.NOT_FOUND:
if self.conf.glance_store.s3_store_create_bucket_on_put:
host = self.conf.glance_store.s3_store_host
location = get_s3_location(host)
try:
s3_conn.create_bucket(bucket, location=location)
except S3ResponseError as e:
msg = (_("Failed to add bucket to S3.\n"
"Got error from S3: %(e)s") % {'e': e})
raise glance.store.BackendException(msg)
else:
msg = (_("The bucket %(bucket)s does not exist in "
"S3. Please set the "
"s3_store_create_bucket_on_put option "
"to add bucket to S3 automatically.")
% {'bucket': bucket})
raise glance.store.BackendException(msg)
def store_add_to_backend(image_id, data, size, store, context=None):
"""
A wrapper around a call to each stores add() method. This gives glance
a common place to check the output
:param image_id: The image add to which data is added
:param data: The data to be stored
:param size: The length of the data in bytes
:param store: The store to which the data is being added
:return: The url location of the file,
the size amount of data,
the checksum of the data
the storage systems metadata dictionary for the location
"""
(location, size, checksum, metadata) = store.add(image_id, data, size)
if metadata is not None:
if not isinstance(metadata, dict):
msg = _(
"The storage driver %(driver)s returned invalid "
" metadata %(metadata)s. This must be a dictionary type"
) % dict(driver=str(store), metadata=str(metadata))
LOG.error(msg)
raise exceptions.BackendException(msg)
try:
check_location_metadata(metadata)
except exceptions.BackendException as e:
e_msg = _(
"A bad metadata structure was returned from the " "%(driver)s storage driver: %(metadata)s. %(e)s."
) % dict(driver=str(store), metadata=str(metadata), e=str(e))
LOG.error(e_msg)
raise exceptions.BackendException(e_msg)
return (location, size, checksum, metadata)
def _create_image_directories(self, directory_paths):
"""
Create directories to write image files if
it does not exist.
:directory_paths is a list of directories belonging to glance store.
:raise BadStoreConfiguration exception if creating a directory fails.
"""
for datadir in directory_paths:
if os.path.exists(datadir):
self._check_write_permission(datadir)
else:
msg = _("Directory to write image files does not exist "
"(%s). Creating.") % datadir
LOG.info(msg)
try:
os.makedirs(datadir)
self._check_write_permission(datadir)
except (IOError, OSError):
if os.path.exists(datadir):
# NOTE(markwash): If the path now exists, some other
# process must have beat us in the race condition.
# But it doesn't hurt, so we can safely ignore
# the error.
self._check_write_permission(datadir)
continue
reason = _("Unable to create datadir: %s") % datadir
LOG.error(reason)
raise exceptions.BadStoreConfiguration(
store_name="filesystem", reason=reason)
def parse_uri(self, uri):
if not uri.startswith('cinder://'):
reason = _("URI must start with cinder://")
LOG.error(reason)
raise exceptions.BadStoreUri(uri, reason)
self.scheme = 'cinder'
self.volume_id = uri[9:]
if not utils.is_uuid_like(self.volume_id):
reason = _("URI contains invalid volume ID: %s") % self.volume_id
LOG.error(reason)
raise exceptions.BadStoreUri(uri, reason)
def add(self, image_id, image_file, image_size, context=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exceptions.Duplicate` if the image already
existed
`glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
loc = StoreLocation({'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id})
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Connection': 'Keep-Alive',
'Cookie': cookie,
'Transfer-Encoding': 'chunked'}
try:
conn = self._get_http_conn('PUT', loc, headers,
content=image_file)
res = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_('Failed to upload content of image '
'%(image)s') % {'image': image_id})
if res.status == httplib.CONFLICT:
raise exceptions.Duplicate(_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = (_('Failed to upload content of image %(image)s') %
{'image': image_id})
LOG.error(msg)
raise exceptions.UnexpectedStatus(status=res.status,
body=res.read())
return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def safe_delete_from_backend(uri, image_id, context=None):
"""Given a uri, delete an image from the store."""
try:
return delete_from_backend(uri, context=context)
except exceptions.NotFound:
msg = _("Failed to delete image %s in store from URI")
LOG.warn(msg % image_id)
except exceptions.StoreDeleteNotSupported as e:
LOG.warn(str(e))
except exceptions.UnsupportedBackend:
exc_type = sys.exc_info()[0].__name__
msg = _("Failed to delete image %(image_id)s " "from store (%(exc_type)s)") % dict(
image_id=image_id, exc_type=exc_type
)
LOG.error(msg)
def _check_context(self, context):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
if context is None:
reason = _("Cinder storage requires a context.")
raise exceptions.BadStoreConfiguration(store_name="cinder",
reason=reason)
if context.service_catalog is None:
reason = _("Cinder storage requires a service catalog.")
raise exceptions.BadStoreConfiguration(store_name="cinder",
reason=reason)
def delete(self, location, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
loc = location.store_location
from boto.s3.connection import S3Connection
uformat = self.conf.glance_store.s3_store_bucket_url_format
calling_format = get_calling_format(s3_store_bucket_url_format=uformat)
s3_conn = S3Connection(loc.accesskey, loc.secretkey,
host=loc.s3serviceurl,
is_secure=(loc.scheme == 's3+https'),
calling_format=calling_format)
bucket_obj = get_bucket(s3_conn, loc.bucket)
# Close the key when we're through.
key = get_key(bucket_obj, loc.key)
msg = _("Deleting image object from S3 using (s3_host=%(s3_host)s, "
"access_key=%(accesskey)s, bucket=%(bucket)s, "
"key=%(obj_name)s)") % ({'s3_host': loc.s3serviceurl,
'accesskey': loc.accesskey,
'bucket': loc.bucket,
'obj_name': loc.key})
LOG.debug(msg)
return key.delete()
def verify_default_store():
scheme = cfg.CONF.glance_store.default_store
try:
get_store_from_scheme(scheme)
except exceptions.UnknownScheme:
msg = _("Store for scheme %s not found") % scheme
raise RuntimeError(msg)
def delete(self, location, context=None):
"""Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
file_path = '[%s] %s' % (
self.datastore_name,
location.store_location.path[len(DS_URL_PREFIX):])
search_index_moref = self._service_content.searchIndex
dc_moref = self._session.invoke_api(self._session.vim,
'FindByInventoryPath',
search_index_moref,
inventoryPath=self.datacenter_path)
delete_task = self._session.invoke_api(
self._session.vim,
'DeleteDatastoreFile_Task',
self._service_content.fileManager,
name=file_path,
datacenter=dc_moref)
try:
self._session.wait_for_task(delete_task)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_('Failed to delete image %(image)s content.') %
{'image': location.image_id})
def _delete_partial(filepath, iid):
try:
os.unlink(filepath)
except Exception as e:
msg = _('Unable to remove partial image '
'data for image %(iid)s: %(e)s')
LOG.error(msg % dict(iid=iid, e=e))
def _option_get(self, param):
result = getattr(self.conf.glance_store, param)
if not result:
reason = (_("Could not find %(param)s in configuration "
"options.") % {'param': param})
raise exceptions.BadStoreConfiguration(
store_name='vmware_datastore', reason=reason)
return result
def configure_add(self):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
if not (self.conf.glance_store.filesystem_store_datadir
or self.conf.glance_store.filesystem_store_datadirs):
reason = (_("Specify at least 'filesystem_store_datadir' or "
"'filesystem_store_datadirs' option"))
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="filesystem",
reason=reason)
if (self.conf.glance_store.filesystem_store_datadir and
self.conf.glance_store.filesystem_store_datadirs):
reason = (_("Specify either 'filesystem_store_datadir' or "
"'filesystem_store_datadirs' option"))
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="filesystem",
reason=reason)
self.multiple_datadirs = False
directory_paths = set()
if self.conf.glance_store.filesystem_store_datadir:
self.datadir = self.conf.glance_store.filesystem_store_datadir
directory_paths.add(self.datadir)
else:
self.multiple_datadirs = True
self.priority_data_map = {}
for datadir in self.conf.glance_store.filesystem_store_datadirs:
(datadir_path,
priority) = self._get_datadir_path_and_priority(datadir)
self._check_directory_paths(datadir_path, directory_paths)
directory_paths.add(datadir_path)
self.priority_data_map.setdefault(int(priority),
[]).append(datadir_path)
self.priority_list = sorted(self.priority_data_map,
reverse=True)
self._create_image_directories(directory_paths)
def add(self, image_id, image_file, image_size, context=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.store.exceptions.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
datadir = self._find_best_datadir(image_size)
filepath = os.path.join(datadir, str(image_id))
if os.path.exists(filepath):
raise exceptions.Duplicate(image=filepath)
checksum = hashlib.md5()
bytes_written = 0
try:
with open(filepath, 'wb') as f:
for buf in utils.chunkreadable(image_file,
ChunkedFile.CHUNKSIZE):
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
except IOError as e:
if e.errno != errno.EACCES:
self._delete_partial(filepath, image_id)
errors = {errno.EFBIG: exceptions.StorageFull(),
errno.ENOSPC: exceptions.StorageFull(),
errno.EACCES: exceptions.StorageWriteDenied()}
raise errors.get(e.errno, e)
except Exception:
with excutils.save_and_reraise_exception():
self._delete_partial(filepath, image_id)
checksum_hex = checksum.hexdigest()
metadata = self._get_metadata()
LOG.debug(_("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s"),
{'bytes_written': bytes_written,
'filepath': filepath,
'checksum_hex': checksum_hex})
return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def _exception_to_unicode(exc):
try:
return unicode(exc)
except UnicodeError:
try:
return strutils.safe_decode(str(exc), errors='ignore')
except UnicodeError:
msg = (_("Caught '%(exception)s' exception.") %
{"exception": exc.__class__.__name__})
return strutils.safe_decode(msg, errors='ignore')
def set_acls(location_uri, public=False, read_tenants=[], write_tenants=None, context=None):
if write_tenants is None:
write_tenants = []
loc = location.get_location_from_uri(location_uri)
scheme = get_store_from_location(location_uri)
store = get_store_from_scheme(scheme)
try:
store.set_acls(loc, public=public, read_tenants=read_tenants, write_tenants=write_tenants)
except NotImplementedError:
LOG.debug(_("Skipping store.set_acls... not implemented."))
def delete(self, location, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if image does not exist
:raises Forbidden if cannot delete because of permissions
"""
loc = location.store_location
fn = loc.path
if os.path.exists(fn):
try:
LOG.debug(_("Deleting image at %(fn)s"), {'fn': fn})
os.unlink(fn)
except OSError:
raise exceptions.Forbidden(_("You cannot delete file %s") % fn)
else:
raise exceptions.NotFound(image=fn)
def _delete_image(self, image_name, snapshot_name=None, context=None):
"""
Delete RBD image and snapshot.
:param image_name Image's name
:param snapshot_name Image snapshot's name
:raises NotFound if image does not exist;
InUseByStore if image is in use or snapshot unprotect failed
"""
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
try:
# First remove snapshot.
if snapshot_name is not None:
with rbd.Image(ioctx, image_name) as image:
try:
image.unprotect_snap(snapshot_name)
except rbd.ImageBusy:
log_msg = _("snapshot %(image)s@%(snap)s "
"could not be unprotected because "
"it is in use")
LOG.debug(log_msg %
{'image': image_name,
'snap': snapshot_name})
raise exceptions.InUseByStore()
image.remove_snap(snapshot_name)
# Then delete image.
rbd.RBD().remove(ioctx, image_name)
except rbd.ImageNotFound:
raise exceptions.NotFound(message=
_("RBD image %s does not exist")
% image_name)
except rbd.ImageBusy:
log_msg = _("image %s could not be removed "
"because it is in use")
LOG.debug(log_msg % image_name)
raise exceptions.InUseByStore()
def _get_file(self, location):
store_location = location
if isinstance(location, glance.store.location.Location):
store_location = location.store_location
try:
parsed = urlparse.urlparse(store_location.get_uri())
return self.fs.get(parsed.netloc)
except gridfs.errors.NoFile:
msg = _("Could not find %s image in GridFS") % \
store_location.get_uri()
LOG.debug(msg)
raise exceptions.NotFound(msg)
def _check_write_permission(self, datadir):
"""
Checks if directory created to write image files has
write permission.
:datadir is a directory path in which glance wites image files.
:raise BadStoreConfiguration exception if datadir is read-only.
"""
if not os.access(datadir, os.W_OK):
msg = (_("Permission to write in %s denied") % datadir)
LOG.exception(msg)
raise exceptions.BadStoreConfiguration(
store_name="filesystem", reason=msg)
def add(self, image_id, image_file, image_size, context=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.store.exceptions.Duplicate` if the image already
existed
"""
loc = StoreLocation({'image_id': image_id})
if self.fs.exists(image_id):
raise exceptions.Duplicate(_("GridFS already has an image at "
"location %s") % loc.get_uri())
LOG.debug(_("Adding a new image to GridFS with "
"id %(iid)s and size %(size)s")
% dict(iid=image_id, size=image_size))
try:
self.fs.put(image_file, _id=image_id)
image = self._get_file(loc)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
self.fs.delete(image_id)
LOG.debug(_("Uploaded image %(iid)s, "
"md5 %(md)s, length %(len)s to GridFS") %
dict(iid=image._id, md=image.md5, len=image.length))
return (loc.get_uri(), image.length, image.md5, {})
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('https', 'http')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path
try:
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
if creds:
try:
self.user, self.password = creds.split(':')
except ValueError:
reason = (_("Credentials '%s' not well-formatted.")
% "".join(creds))
LOG.debug(reason)
raise exceptions.BadStoreUri()
else:
self.user = None
if netloc == '':
LOG.debug(_("No address specified in HTTP URL"))
raise exceptions.BadStoreUri(uri=uri)
self.netloc = netloc
self.path = path
def configure_add(self):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
try:
self.chunk_size = CONF.sheepdog_store_chunk_size * units.Mi
self.addr = CONF.sheepdog_store_address
self.port = CONF.sheepdog_store_port
except cfg.ConfigFileValueError as e:
reason = _("Error in store configuration: %s") % e
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="sheepdog", reason=reason)
try:
processutils.execute("collie", shell=True)
except processutils.ProcessExecutionError as exc:
reason = _("Error in store configuration: %s") % exc
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="sheepdog", reason=reason)
def get(self, location, offset=0, chunk_size=None, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises `glance.store.exceptions.NotFound` if image does not exist
"""
filepath, filesize = self._resolve_location(location)
msg = _("Found image at %s. Returning in ChunkedFile.") % filepath
LOG.debug(msg)
return (ChunkedFile(filepath), filesize)
def get_size(self, location, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file and returns the image size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises `glance.store.exceptions.NotFound` if image does not exist
:rtype int
"""
filepath, filesize = self._resolve_location(location)
msg = _("Found image at %s.") % filepath
LOG.debug(msg)
return filesize
def _query(self, location, verb, depth=0):
if depth > MAX_REDIRECTS:
reason = (_("The HTTP URL exceeded %s maximum "
"redirects.") % MAX_REDIRECTS)
LOG.debug(reason)
raise exceptions.MaxRedirectsExceeded(message=reason)
loc = location.store_location
conn_class = self._get_conn_class(loc)
conn = conn_class(loc.netloc)
conn.request(verb, loc.path, "", {})
resp = conn.getresponse()
# Check for bad status codes
if resp.status >= 400:
reason = (_("HTTP URL %(url)s returned a "
"%(status)s status code.") %
dict(url=loc.path, status=resp.status))
LOG.debug(reason)
raise exceptions.BadStoreUri(message=reason)
location_header = resp.getheader("location")
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(url)s attempted to redirect "
"with an invalid %(status)s status code.") %
dict(url=loc.path, status=resp.status))
LOG.debug(reason)
raise exceptions.BadStoreUri(message=reason)
location_class = glance.store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, verb, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('file', 'filesystem')
self.scheme = pieces.scheme
path = (pieces.netloc + pieces.path).strip()
if path == '':
reason = _("No path specified in URI: %s") % uri
LOG.debug(reason)
raise exceptions.BadStoreUri('No path specified')
self.path = path
