def parse_uri(self, uri):
prefix = "rbd://"
if not uri.startswith(prefix):
reason = _("URI must start with rbd://")
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
# convert to ascii since librbd doesn't handle unicode
try:
ascii_uri = str(uri)
except UnicodeError:
reason = _("URI contains non-ascii characters")
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
pieces = ascii_uri[len(prefix) :].split("/")
if len(pieces) == 1:
self.fsid, self.pool, self.image, self.snapshot = (None, None, pieces[0], None)
elif len(pieces) == 4:
self.fsid, self.pool, self.image, self.snapshot = map(urllib.unquote, pieces)
else:
reason = _("URI must have exactly 1 or 4 components")
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
if any(map(lambda p: p == "", pieces)):
reason = _("URI cannot contain empty components")
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
def add(self, image_id, image_file, image_size, context=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance_store.exceptions.Duplicate` if the image already
existed
S3 writes the image data using the scheme:
s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ>
where:
<USER> = ``s3_store_user``
<KEY> = ``s3_store_key``
<S3_HOST> = ``s3_store_host``
<BUCKET> = ``s3_store_bucket``
<ID> = The id of the image being added
"""
loc = StoreLocation(
{
"scheme": self.scheme,
"bucket": self.bucket,
"key": image_id,
"s3serviceurl": self.full_s3_host,
"accesskey": self.access_key,
"secretkey": self.secret_key,
},
self.conf,
)
s3_conn = self._create_connection(loc)
create_bucket_if_missing(self.conf, self.bucket, s3_conn)
bucket_obj = get_bucket(s3_conn, self.bucket)
obj_name = str(image_id)
key = bucket_obj.get_key(obj_name)
if key and key.exists():
raise exceptions.Duplicate(
message=_("S3 already has an image at " "location %s") % self._sanitize(loc.get_uri())
)
msg = _(
"Adding image object to S3 using (s3_host=%(s3_host)s, "
"access_key=%(access_key)s, bucket=%(bucket)s, "
"key=%(obj_name)s)"
) % ({"s3_host": self.s3_host, "access_key": self.access_key, "bucket": self.bucket, "obj_name": obj_name})
LOG.debug(msg)
LOG.debug("Uploading an image file to S3 for %s" % self._sanitize(loc.get_uri()))
if image_size < self.s3_store_large_object_size:
return self.add_singlepart(image_file, bucket_obj, obj_name, loc)
else:
return self.add_multipart(image_file, image_size, bucket_obj, obj_name, loc)
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file and returns the image size
:param location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises `glance_store.exceptions.NotFound` if image does not exist
:rtype int
"""
loc = location.store_location
try:
self._check_context(context)
volume = get_cinderclient(self.conf,
context).volumes.get(loc.volume_id)
# GB unit convert to byte
return volume.size * (1024 ** 3)
except cinder_exception.NotFound as e:
reason = _("Failed to get image size due to "
"volume can not be found: %s") % self.volume_id
LOG.error(reason)
raise exceptions.NotFound(reason)
except Exception as e:
LOG.exception(_("Failed to get image size due to "
"internal error: %s") % e)
return 0
def _sanity_check(self):
if self.conf.glance_store.vmware_api_retry_count <= 0:
msg = _('vmware_api_retry_count should be greater than zero')
LOG.error(msg)
raise exceptions.BadStoreConfiguration(
store_name='vmware_datastore', reason=msg)
if self.conf.glance_store.vmware_task_poll_interval <= 0:
msg = _('vmware_task_poll_interval should be greater than zero')
LOG.error(msg)
raise exceptions.BadStoreConfiguration(
store_name='vmware_datastore', reason=msg)
if not (self.conf.glance_store.vmware_datastore_name
or self.conf.glance_store.vmware_datastores):
msg = (_("Specify at least 'vmware_datastore_name' or "
"'vmware_datastores' option"))
LOG.error(msg)
raise exceptions.BadStoreConfiguration(
store_name='vmware_datastore', reason=msg)
if (self.conf.glance_store.vmware_datastore_name and
self.conf.glance_store.vmware_datastores):
msg = (_("Specify either 'vmware_datastore_name' or "
"'vmware_datastores' option"))
LOG.error(msg)
raise exceptions.BadStoreConfiguration(
store_name='vmware_datastore', reason=msg)
def get(self, location, offset=0, chunk_size=None, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:param offset: offset to start reading
:param chunk_size: size to read, or None to get all the image
:param context: Request context
:raises `glance_store.exceptions.NotFound` if image does not exist
"""
loc = location.store_location
self._check_context(context)
try:
client = get_cinderclient(self.conf, context)
volume = client.volumes.get(loc.volume_id)
size = int(volume.metadata.get('image_size',
volume.size * units.Gi))
iterator = self._cinder_volume_data_iterator(
client, volume, size, offset=offset,
chunk_size=self.READ_CHUNKSIZE, partial_length=chunk_size)
return (iterator, chunk_size or size)
except cinder_exception.NotFound:
reason = _("Failed to get image size due to "
"volume can not be found: %s") % volume.id
LOG.error(reason)
raise exceptions.NotFound(reason)
except cinder_exception.ClientException as e:
msg = (_('Failed to get image volume %(volume_id): %(error)s')
% {'volume_id': loc.volume_id, 'error': e})
LOG.error(msg)
raise exceptions.BackendException(msg)
def _delete_image(self, image_name, snapshot_name=None, context=None):
"""
Delete RBD image and snapshot.
:param image_name Image's name
:param snapshot_name Image snapshot's name
:raises NotFound if image does not exist;
InUseByStore if image is in use or snapshot unprotect failed
"""
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
try:
# First remove snapshot.
if snapshot_name is not None:
with rbd.Image(ioctx, image_name) as image:
try:
image.unprotect_snap(snapshot_name)
except rbd.ImageBusy:
log_msg = _(
"snapshot %(image)s@%(snap)s " "could not be unprotected because " "it is in use"
)
LOG.debug(log_msg % {"image": image_name, "snap": snapshot_name})
raise exceptions.InUseByStore()
image.remove_snap(snapshot_name)
# Then delete image.
rbd.RBD().remove(ioctx, image_name)
except rbd.ImageNotFound:
raise exceptions.NotFound(message=_("RBD image %s does not exist") % image_name)
except rbd.ImageBusy:
log_msg = _("image %s could not be removed " "because it is in use")
LOG.debug(log_msg % image_name)
raise exceptions.InUseByStore()
def parse_uri(self, uri):
prefix = 'rbd://'
self.validate_schemas(uri, valid_schemas=(prefix,))
# convert to ascii since librbd doesn't handle unicode
try:
ascii_uri = str(uri)
except UnicodeError:
reason = _('URI contains non-ascii characters')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
pieces = ascii_uri[len(prefix):].split('/')
if len(pieces) == 1:
self.fsid, self.pool, self.image, self.snapshot = \
(None, None, pieces[0], None)
elif len(pieces) == 4:
self.fsid, self.pool, self.image, self.snapshot = \
map(urllib.parse.unquote, pieces)
else:
reason = _('URI must have exactly 1 or 4 components')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
if any(map(lambda p: p == '', pieces)):
reason = _('URI cannot contain empty components')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
def _wait_volume_status(self, volume, status_transition, status_expected):
max_recheck_wait = 15
timeout = self.conf.glance_store.cinder_state_transition_timeout
volume = volume.manager.get(volume.id)
tries = 0
elapsed = 0
while volume.status == status_transition:
if elapsed >= timeout:
msg = (_('Timeout while waiting while volume %(volume_id)s '
'status is %(status)s.')
% {'volume_id': volume.id, 'status': status_transition})
LOG.error(msg)
raise exceptions.BackendException(msg)
wait = min(0.5 * 2 ** tries, max_recheck_wait)
time.sleep(wait)
tries += 1
elapsed += wait
volume = volume.manager.get(volume.id)
if volume.status != status_expected:
msg = (_('The status of volume %(volume_id)s is unexpected: '
'status = %(status)s, expected = %(expected)s.')
% {'volume_id': volume.id, 'status': volume.status,
'expected': status_expected})
LOG.error(msg)
raise exceptions.BackendException(msg)
return volume
def _load_config(self):
try:
scf = self.conf.glance_store.swift_store_config_file
conf_file = self.conf.find_file(scf)
CONFIG.read(conf_file)
except Exception as e:
msg = (i18n._("swift config file "
"%(conf)s:%(exc)s not found") %
{'conf': self.conf.glance_store.swift_store_config_file,
'exc': e})
LOG.error(msg)
raise exceptions.BadStoreConfiguration(store_name='swift',
reason=msg)
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
reference['auth_address'] = CONFIG.get(ref, 'auth_address')
reference['user'] = CONFIG.get(ref, 'user')
reference['key'] = CONFIG.get(ref, 'key')
account_params[ref] = reference
except (ValueError, SyntaxError, configparser.NoOptionError) as e:
LOG.exception(i18n._("Invalid format of swift store config"
"cfg"))
return account_params
def store_add_to_backend(image_id, data, size, store, context=None):
"""
A wrapper around a call to each stores add() method. This gives glance
a common place to check the output
:param image_id: The image add to which data is added
:param data: The data to be stored
:param size: The length of the data in bytes
:param store: The store to which the data is being added
:return: The url location of the file,
the size amount of data,
the checksum of the data
the storage systems metadata dictionary for the location
"""
(location, size, checksum, metadata) = store.add(image_id, data, size)
if metadata is not None:
if not isinstance(metadata, dict):
msg = (_("The storage driver %(driver)s returned invalid "
" metadata %(metadata)s. This must be a dictionary type")
% dict(driver=str(store), metadata=str(metadata)))
LOG.error(msg)
raise exceptions.BackendException(msg)
try:
check_location_metadata(metadata)
except exceptions.BackendException as e:
e_msg = (_("A bad metadata structure was returned from the "
"%(driver)s storage driver: %(metadata)s. %(e)s.") %
dict(driver=unicode(store),
metadata=unicode(metadata),
e=unicode(e)))
LOG.error(e_msg)
raise exceptions.BackendException(e_msg)
return (location, size, checksum, metadata)
def create_bucket_if_missing(conf, bucket, s3_conn):
"""
Creates a missing bucket in S3 if the
``s3_store_create_bucket_on_put`` option is set.
:param conf: Configuration
:param bucket: Name of bucket to create
:param s3_conn: Connection to S3
"""
from boto.exception import S3ResponseError
try:
s3_conn.get_bucket(bucket)
except S3ResponseError as e:
if e.status == httplib.NOT_FOUND:
if conf.glance_store.s3_store_create_bucket_on_put:
host = conf.glance_store.s3_store_host
location = get_s3_location(host)
try:
s3_conn.create_bucket(bucket, location=location)
except S3ResponseError as e:
msg = (_("Failed to add bucket to S3.\n"
"Got error from S3: %s.") %
utils.exception_to_str(e))
raise glance_store.BackendException(msg)
else:
msg = (_("The bucket %(bucket)s does not exist in "
"S3. Please set the "
"s3_store_create_bucket_on_put option "
"to add bucket to S3 automatically.")
% {'bucket': bucket})
raise glance_store.BackendException(msg)
def _create_image_directories(self, directory_paths):
"""
Create directories to write image files if
it does not exist.
:directory_paths is a list of directories belonging to glance store.
:raises: BadStoreConfiguration exception if creating a directory fails.
"""
for datadir in directory_paths:
if os.path.exists(datadir):
self._check_write_permission(datadir)
self._set_exec_permission(datadir)
else:
msg = _("Directory to write image files does not exist "
"(%s). Creating.") % datadir
LOG.info(msg)
try:
os.makedirs(datadir)
self._check_write_permission(datadir)
self._set_exec_permission(datadir)
except (IOError, OSError):
if os.path.exists(datadir):
# NOTE(markwash): If the path now exists, some other
# process must have beat us in the race condition.
# But it doesn't hurt, so we can safely ignore
# the error.
self._check_write_permission(datadir)
self._set_exec_permission(datadir)
continue
reason = _("Unable to create datadir: %s") % datadir
LOG.error(reason)
raise exceptions.BadStoreConfiguration(
store_name="filesystem", reason=reason)
def _get_datadir_path_and_priority(self, datadir):
"""
Gets directory paths and its priority from
filesystem_store_datadirs option in glance-api.conf.
:param datadir: is directory path with its priority.
:returns: datadir_path as directory path
priority as priority associated with datadir_path
:raises: BadStoreConfiguration exception if priority is invalid or
empty directory path is specified.
"""
priority = 0
parts = [part.strip() for part in datadir.rsplit(":", 1)]
datadir_path = parts[0]
if len(parts) == 2 and parts[1]:
priority = parts[1]
if not priority.isdigit():
msg = (_("Invalid priority value %(priority)s in "
"filesystem configuration") % {'priority': priority})
LOG.exception(msg)
raise exceptions.BadStoreConfiguration(
store_name="filesystem", reason=msg)
if not datadir_path:
msg = _("Invalid directory specified in filesystem configuration")
LOG.exception(msg)
raise exceptions.BadStoreConfiguration(
store_name="filesystem", reason=msg)
return datadir_path, priority
def configure_add(self):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
try:
chunk_size = self.conf.glance_store.sheepdog_store_chunk_size
self.chunk_size = chunk_size * units.Mi
self.READ_CHUNKSIZE = self.chunk_size
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
self.addr = self.conf.glance_store.sheepdog_store_address
self.port = self.conf.glance_store.sheepdog_store_port
except cfg.ConfigFileValueError as e:
reason = _("Error in store configuration: %s") % e
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name='sheepdog',
reason=reason)
try:
processutils.execute("collie", shell=True)
except processutils.ProcessExecutionError as exc:
reason = _("Error in store configuration: %s") % exc
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name='sheepdog',
reason=reason)
def _parse_datastore_info_and_weight(self, datastore):
weight = 0
parts = map(lambda x: x.strip(), datastore.rsplit(":", 2))
if len(parts) < 2:
msg = _('vmware_datastores format must be '
'datacenter_path:datastore_name:weight or '
'datacenter_path:datastore_name')
LOG.error(msg)
raise exceptions.BadStoreConfiguration(
store_name='vmware_datastore', reason=msg)
if len(parts) == 3 and parts[2]:
weight = parts[2]
if not weight.isdigit():
msg = (_('Invalid weight value %(weight)s in '
'vmware_datastores configuration') %
{'weight': weight})
LOG.exception(msg)
raise exceptions.BadStoreConfiguration(
store_name="vmware_datastore", reason=msg)
datacenter_path, datastore_name = parts[0], parts[1]
if not datacenter_path or not datastore_name:
msg = _('Invalid datacenter_path or datastore_name specified '
'in vmware_datastores configuration')
LOG.exception(msg)
raise exceptions.BadStoreConfiguration(
store_name="vmware_datastore", reason=msg)
return datacenter_path, datastore_name, weight
def _sanity_check(self):
if self.conf.glance_store.vmware_api_retry_count <= 0:
msg = _("vmware_api_retry_count should be greater than zero")
LOG.error(msg)
raise exceptions.BadStoreConfiguration(store_name="vmware_datastore", reason=msg)
if self.conf.glance_store.vmware_task_poll_interval <= 0:
msg = _("vmware_task_poll_interval should be greater than zero")
LOG.error(msg)
raise exceptions.BadStoreConfiguration(store_name="vmware_datastore", reason=msg)
def _query(self, location, verb, depth=0):
if depth > MAX_REDIRECTS:
reason = (_("The HTTP URL exceeded %s maximum "
"redirects.") % MAX_REDIRECTS)
LOG.debug(reason)
raise exceptions.MaxRedirectsExceeded(message=reason)
loc = location.store_location
conn_class = self._get_conn_class(loc)
conn = conn_class(loc.netloc)
hearders = {}
if loc.token:
# hearders.setdefault('x-auth-token', loc.token)
# verb = 'GET'
# conn.request(verb, loc.path, "", hearders)
# resp = conn.getresponse()
# try:
# size = jsonutils.loads(resp.read())['size']
# except Exception:
# size = 0
# raise exception.BadStoreUri(loc.path, reason)
return (conn, None, 1)
conn.request(verb, loc.path, "", {})
resp = conn.getresponse()
# Check for bad status codes
if resp.status >= 400:
if resp.status == httplib.NOT_FOUND:
reason = _("HTTP datastore could not find image at URI.")
LOG.debug(reason)
raise exceptions.NotFound(message=reason)
reason = (_("HTTP URL %(url)s returned a "
"%(status)s status code.") %
dict(url=loc.path, status=resp.status))
LOG.debug(reason)
raise exceptions.BadStoreUri(message=reason)
location_header = resp.getheader("location")
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(url)s attempted to redirect "
"with an invalid %(status)s status code.") %
dict(url=loc.path, status=resp.status))
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
location_class = glance_store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, verb, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def configure_add(self):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
self.s3_host = self._option_get('s3_store_host')
access_key = self._option_get('s3_store_access_key')
secret_key = self._option_get('s3_store_secret_key')
if six.PY2:
# NOTE(jaypipes): Need to encode to UTF-8 here because of a
# bug in the HMAC library that boto uses.
# See: http://bugs.python.org/issue5285
# See: http://trac.edgewall.org/ticket/8083
access_key = access_key.encode('utf-8')
secret_key = secret_key.encode('utf-8')
self.access_key = access_key
self.secret_key = secret_key
self.bucket = self._option_get('s3_store_bucket')
self.scheme = 's3'
if self.s3_host.startswith('https://'):
self.scheme = 's3+https'
self.full_s3_host = self.s3_host
elif self.s3_host.startswith('http://'):
self.full_s3_host = self.s3_host
else: # Defaults http
self.full_s3_host = 'http://' + self.s3_host
buffer_dir = self.conf.glance_store.s3_store_object_buffer_dir
self.s3_store_object_buffer_dir = buffer_dir
_s3_obj_size = self._option_get('s3_store_large_object_size')
self.s3_store_large_object_size = _s3_obj_size * units.Mi
_s3_ck_size = self._option_get('s3_store_large_object_chunk_size')
_s3_ck_min = DEFAULT_LARGE_OBJECT_MIN_CHUNK_SIZE
if _s3_ck_size < _s3_ck_min:
reason = (_("s3_store_large_object_chunk_size must be at "
"least %(_s3_ck_min)d MB. "
"You configured it as %(_s3_ck_size)d MB") %
{'_s3_ck_min': _s3_ck_min,
'_s3_ck_size': _s3_ck_size})
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="s3",
reason=reason)
self.s3_store_large_object_chunk_size = _s3_ck_size * units.Mi
self.s3_store_thread_pools = self._option_get('s3_store_thread_pools')
if self.s3_store_thread_pools <= 0:
reason = (_("s3_store_thread_pools must be a positive "
"integer. %s") % self.s3_store_thread_pools)
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="s3",
reason=reason)
def verify_store():
store_id = CONF.glance_store.default_backend
if not store_id:
msg = _("'default_backend' config option is not set.")
raise RuntimeError(msg)
try:
get_store_from_store_identifier(store_id)
except exceptions.UnknownScheme:
msg = _("Store for identifier %s not found") % store_id
raise RuntimeError(msg)
def _check_context(self, context, require_tenant=False):
user_overriden = is_user_overriden(self.conf)
if user_overriden and not require_tenant:
return
if context is None:
reason = _("Cinder storage requires a context.")
raise exceptions.BadStoreConfiguration(store_name="cinder",
reason=reason)
if not user_overriden and context.service_catalog is None:
reason = _("Cinder storage requires a service catalog.")
raise exceptions.BadStoreConfiguration(store_name="cinder",
reason=reason)
def _delete_image(self, target_pool, image_name,
snapshot_name=None, context=None):
"""
Delete RBD image and snapshot.
:param image_name: Image's name
:param snapshot_name: Image snapshot's name
:raises: NotFound if image does not exist;
InUseByStore if image is in use or snapshot unprotect failed
"""
with self.get_connection(conffile=self.conf_file,
rados_id=self.user) as conn:
with conn.open_ioctx(target_pool) as ioctx:
try:
# First remove snapshot.
if snapshot_name is not None:
with rbd.Image(ioctx, image_name) as image:
try:
self._unprotect_snapshot(image, snapshot_name)
image.remove_snap(snapshot_name)
except rbd.ImageNotFound as exc:
msg = (_("Snap Operating Exception "
"%(snap_exc)s "
"Snapshot does not exist.") %
{'snap_exc': exc})
LOG.debug(msg)
except rbd.ImageBusy as exc:
log_msg = (_LE("Snap Operating Exception "
"%(snap_exc)s "
"Snapshot is in use.") %
{'snap_exc': exc})
LOG.error(log_msg)
raise exceptions.InUseByStore()
# Then delete image.
rbd.RBD().remove(ioctx, image_name)
except rbd.ImageHasSnapshots:
log_msg = (_LE("Remove image %(img_name)s failed. "
"It has snapshot(s) left.") %
{'img_name': image_name})
LOG.error(log_msg)
raise exceptions.HasSnapshot()
except rbd.ImageBusy:
log_msg = (_LE("Remove image %(img_name)s failed. "
"It is in use.") %
{'img_name': image_name})
LOG.error(log_msg)
raise exceptions.InUseByStore()
except rbd.ImageNotFound:
msg = _("RBD image %s does not exist") % image_name
raise exceptions.NotFound(message=msg)
def parse_uri(self, uri):
if not uri.startswith('cinder://'):
reason = _("URI must start with 'cinder://'")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
self.scheme = 'cinder'
self.volume_id = uri[9:]
if not utils.is_uuid_like(self.volume_id):
reason = _("URI contains invalid volume ID")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
def _query(self, location, method, depth=0):
if depth > MAX_REDIRECTS:
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.", {'max_redirects': MAX_REDIRECTS})
LOG.debug(msg)
raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
loc = location.store_location
# NOTE(arnaud): use a decorator when the config is not tied to self
for i in range(self.api_retry_count + 1):
cookie = self._build_vim_cookie_header(
self.session.vim.client.options.transport.cookiejar)
headers = {'Cookie': cookie}
try:
conn = self._get_http_conn(method, loc, headers)
resp = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to access image %(image)s '
'content.') % {'image':
location.image_id})
if resp.status >= 400:
if resp.status == httplib.UNAUTHORIZED:
self.reset_session(force=True)
continue
if resp.status == httplib.NOT_FOUND:
reason = _('VMware datastore could not find image at URI.')
LOG.info(reason)
raise exceptions.NotFound(message=reason)
msg = ('HTTP request returned a %(status)s status code.'
% {'status': resp.status})
LOG.debug(msg)
raise exceptions.BadStoreUri(msg)
break
location_header = resp.getheader('location')
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.")
% {'path': loc.path, 'status': resp.status})
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
location_class = glance_store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, method, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def add(self, image_id, image_file, image_size, context=None):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exceptions.Duplicate` if the image already
existed
`glance.common.exceptions.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
loc = StoreLocation({'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id})
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Connection': 'Keep-Alive',
'Cookie': cookie,
'Transfer-Encoding': 'chunked'}
try:
conn = self._get_http_conn('PUT', loc, headers,
content=image_file)
res = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_('Failed to upload content of image '
'%(image)s') % {'image': image_id})
if res.status == httplib.CONFLICT:
raise exceptions.Duplicate(_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = (_('Failed to upload content of image %(image)s') %
{'image': image_id})
LOG.error(msg)
raise exceptions.UnexpectedStatus(status=res.status,
body=res.read())
return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def safe_delete_from_backend(uri, image_id, context=None):
"""Given a uri, delete an image from the store."""
try:
return delete_from_backend(uri, context=context)
except exceptions.NotFound:
msg = _('Failed to delete image %s in store from URI')
LOG.warn(msg % image_id)
except exceptions.StoreDeleteNotSupported as e:
LOG.warn(str(e))
except exceptions.UnsupportedBackend:
exc_type = sys.exc_info()[0].__name__
msg = (_('Failed to delete image %(image_id)s '
'from store (%(exc_type)s)') %
dict(image_id=image_id, exc_type=exc_type))
LOG.error(msg)
def connect_and_confirm(self):
"""
Confirm connection to irods with the given credentials
"""
try:
msg = (_("connecting to irods://%(host)s:%(port)s%(path)s " +
"in zone %(zone)s") %
({'host': self.host, 'port': self.port,
'path': self.datastore, 'zone': self.zone}))
LOG.debug(msg)
sess = iRODSSession(user=str(self.user), password=str(self.password), host=str(
self.host), port=int(self.port), zone=str(self.zone))
coll = sess.collections.get(self.test_path)
except Exception as e:
reason = (_("Could not connect with host, port, zone," +
" path, or user/password. %s" % (e)))
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="irods",
reason=reason)
else:
self.irods_conn_object = sess
self.irods_conn_object.cleanup()
finally:
sess.cleanup()
# check if file exists
LOG.debug(_("attempting to check the collection %s" % self.datastore))
try:
coll = sess.collections.get(self.datastore)
except Exception:
reason = _("collection '%s' is not valid or must be " +
"created beforehand") % self.datastore
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="irods",
reason=reason)
finally:
sess.cleanup()
LOG.debug(_("success"))
return True
def delete(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file to delete
:location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
loc = location.store_location
s3host, s3port = netutils.parse_host_port(loc.s3serviceurl, 80)
from boto.s3.connection import S3Connection
uformat = self.conf.glance_store.s3_store_bucket_url_format
calling_format = get_calling_format(s3_store_bucket_url_format=uformat)
s3_conn = S3Connection(loc.accesskey, loc.secretkey,
host=s3host, port=s3port,
is_secure=(loc.scheme == 's3+https'),
calling_format=calling_format)
bucket_obj = get_bucket(s3_conn, loc.bucket)
# Close the key when we're through.
key = get_key(bucket_obj, loc.key)
msg = _("Deleting image object from S3 using (s3_host=%(s3_host)s, "
"access_key=%(accesskey)s, bucket=%(bucket)s, "
"key=%(obj_name)s)") % ({'s3_host': loc.s3serviceurl,
'accesskey': loc.accesskey,
'bucket': loc.bucket,
'obj_name': loc.key})
LOG.debug(msg)
return key.delete()
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns the size
:param location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises `glance_store.exceptions.NotFound` if image does not exist
"""
loc = location.store_location
# if there is a pool specific in the location, use it; otherwise
# we fall back to the default pool specified in the config
target_pool = loc.pool or self.pool
with rados.Rados(conffile=self.conf_file,
rados_id=self.user) as conn:
with conn.open_ioctx(target_pool) as ioctx:
try:
with rbd.Image(ioctx, loc.image,
snapshot=loc.snapshot) as image:
img_info = image.stat()
return img_info['size']
except rbd.ImageNotFound:
msg = _('RBD image %s does not exist') % loc.get_uri()
LOG.debug(msg)
raise exceptions.NotFound(msg)
def _check_context(self, context):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
if context is None:
reason = _("Cinder storage requires a context.")
raise exceptions.BadStoreConfiguration(store_name="cinder",
reason=reason)
if context.service_catalog is None:
reason = _("Cinder storage requires a service catalog.")
raise exceptions.BadStoreConfiguration(store_name="cinder",
reason=reason)
def verify_default_store():
scheme = CONF.glance_store.default_store
try:
get_store_from_scheme(scheme)
except exceptions.UnknownScheme:
msg = _("Store for scheme %s not found") % scheme
raise RuntimeError(msg)
def get_cinderclient(conf, context=None):
glance_store = conf.glance_store
user_overriden = is_user_overriden(conf)
if user_overriden:
username = glance_store.cinder_store_user_name
password = glance_store.cinder_store_password
project = glance_store.cinder_store_project_name
url = glance_store.cinder_store_auth_address
else:
username = context.user
password = context.auth_token
project = context.tenant
if glance_store.cinder_endpoint_template:
url = glance_store.cinder_endpoint_template % context.to_dict()
else:
info = glance_store.cinder_catalog_info
service_type, service_name, interface = info.split(':')
try:
catalog = keystone_sc.ServiceCatalogV2(context.service_catalog)
url = catalog.url_for(
region_name=glance_store.cinder_os_region_name,
service_type=service_type,
service_name=service_name,
interface=interface)
except keystone_exc.EndpointNotFound:
reason = _("Failed to find Cinder from a service catalog.")
raise exceptions.BadStoreConfiguration(store_name="cinder",
reason=reason)
c = cinderclient.Client(username,
password,
project,
auth_url=url,
insecure=glance_store.cinder_api_insecure,
retries=glance_store.cinder_http_retries,
cacert=glance_store.cinder_ca_certificates_file)
LOG.debug('Cinderclient connection created for user %(user)s using URL: '
'%(url)s.', {'user': username, 'url': url})
# noauth extracts user_id:project_id from auth_token
if not user_overriden:
c.client.auth_token = context.auth_token or '%s:%s' % (username,
project)
c.client.management_url = url
return c
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urllib.parse.urlparse(uri)
assert pieces.scheme in ('https', 'http')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path
try:
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
if creds:
try:
self.user, self.password = creds.split(':')
except ValueError:
reason = _("Credentials are not well-formatted.")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
else:
self.user = None
if netloc == '':
LOG.info(_LI("No address specified in HTTP URL"))
raise exceptions.BadStoreUri(uri=uri)
else:
# IPv6 address has the following format [1223:0:0:..]:<some_port>
# we need to be sure that we are validating port in both IPv4,IPv6
delimiter = "]:" if netloc.count(":") > 1 else ":"
host, dlm, port = netloc.partition(delimiter)
# if port is present in location then validate port format
if port and not port.isdigit():
raise exceptions.BadStoreUri(uri=uri)
self.netloc = netloc
self.path = path
def get(self, location, offset=0, chunk_size=None, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises: `glance_store.exceptions.NotFound` if image does not exist
"""
filepath, filesize = self._resolve_location(location)
msg = _("Found image at %s. Returning in ChunkedFile.") % filepath
LOG.debug(msg)
return (ChunkedFile(filepath,
offset=offset,
chunk_size=self.READ_CHUNKSIZE,
partial_length=chunk_size), chunk_size or filesize)
def register_store_opts(conf):
for store_entry in set(conf.glance_store.stores):
LOG.debug("Registering options for %s" % store_entry)
store_cls = _load_store(conf, store_entry, False)
if store_cls is None:
msg = _('Store %s not found') % store_entry
raise exceptions.GlanceStoreException(message=msg)
if getattr(store_cls, 'OPTIONS', None) is not None:
# NOTE(flaper87): To be removed in k-2. This should
# give deployers enough time to migrate their systems
# and move configs under the new section.
for opt in store_cls.OPTIONS:
opt.deprecated_opts = [cfg.DeprecatedOpt(opt.name,
group='DEFAULT')]
conf.register_opt(opt, group=_STORE_CFG_GROUP)
def get_store_from_store_identifier(store_identifier):
"""Determine backing store from identifier.
Given a store identifier, return the appropriate store object
for handling that scheme.
"""
scheme_map = {}
enabled_backends = CONF.enabled_backends
try:
scheme = enabled_backends[store_identifier]
except KeyError:
msg = _("Store for identifier %s not found") % store_identifier
raise exceptions.UnknownScheme(msg)
if scheme not in location.SCHEME_TO_CLS_BACKEND_MAP:
raise exceptions.UnknownScheme(scheme=scheme)
scheme_info = location.SCHEME_TO_CLS_BACKEND_MAP[scheme][store_identifier]
store = scheme_info['store']
if not store.is_capable(capabilities.BitMasks.DRIVER_REUSABLE):
# Driver instance isn't stateless so it can't
# be reused safely and need recreation.
store_entry = scheme_info['store_entry']
store = _load_multi_store(store.conf,
store_entry,
invoke_load=True,
backend=store_identifier)
store.configure()
try:
loc_cls = store.get_store_location_class()
for new_scheme in store.get_schemes():
if new_scheme not in scheme_map:
scheme_map[new_scheme] = {}
scheme_map[new_scheme][store_identifier] = {
'store': store,
'location_class': loc_cls,
'store_entry': store_entry
}
location.register_scheme_backend_map(scheme_map)
except NotImplementedError:
scheme_info['store'] = store
return store
def get(self, location, offset=0, chunk_size=None, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises: `glance_store.exceptions.NotFound` if image does not exist
"""
loc = location.store_location
image = SheepdogImage(loc.addr, loc.port, loc.image,
self.READ_CHUNKSIZE)
if not image.exist():
raise exceptions.NotFound(
_("Sheepdog image %s does not exist") % image.name)
return (ImageIterator(image), image.get_size())
def parse_uri(self, uri):
"""Parse URLs based on GCE scheme """
LOG.debug('Parse uri %s' % (uri, ))
if not uri.startswith('%s://' % STORE_SCHEME):
reason = (_("URI %(uri)s must start with %(scheme)s://") % {
'uri': uri,
'scheme': STORE_SCHEME
})
LOG.error(reason)
raise exceptions.BadStoreUri(message=reason)
pieces = urllib.parse.urlparse(uri)
self.scheme = pieces.scheme
gce_project = pieces.netloc
gce_id, glance_id = pieces.path.strip('/').split('/')
parse_params = (gce_project, gce_id, glance_id)
if not all([parse_params]):
raise exceptions.BadStoreUri(uri=uri)
self.gce_project, self.gce_id, self.glance_id = parse_params
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file and returns the image size
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises: `glance_store.exceptions.NotFound` if image does not exist
:rtype: int
"""
loc = location.store_location
image = SheepdogImage(loc.addr, loc.port, loc.image,
self.READ_CHUNKSIZE)
if not image.exist():
raise exceptions.NotFound(
_("Sheepdog image %s does not exist") % image.name)
return image.get_size()
def __iter__(self):
try:
with self.store.get_connection(conffile=self.conf_file,
rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
with rbd.Image(ioctx, self.name,
snapshot=self.snapshot) as image:
size = image.size()
bytes_left = size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image.read(size - bytes_left, length)
bytes_left -= len(data)
yield data
raise StopIteration()
except rbd.ImageNotFound:
raise exceptions.NotFound(
_('RBD image %s does not exist') % self.name)
def __iter__(self):
try:
with rados.Rados(conffile=self.conf_file,
rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
with rbd.Image(ioctx, self.name) as image:
img_info = image.stat()
size = img_info['size']
bytes_left = size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image.read(size - bytes_left, length)
bytes_left -= len(data)
yield data
raise StopIteration()
except rbd.ImageNotFound:
raise exceptions.NotFound(
_('RBD image %s does not exist') % self.name)
def delete(self, location, context=None):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if image does not exist
:raises Forbidden if cannot delete because of permissions
"""
full_data_path = self.path + "/" + location.store_location.data_name
LOG.debug(
_("connecting to %(host)s for %(data)s" % ({
'host': self.host,
'data': full_data_path
})))
self.irods_manager.delete_image_file(full_data_path)
def get_key(bucket, obj):
"""
Get a key from a bucket
:param bucket: The ``boto.s3.Bucket``
:param obj: Object to get the key for
:raises ``glance_store.exceptions.NotFound`` if key is not found.
"""
key = bucket.get_key(obj)
if not key or not key.exists():
msg = (_("Could not find key %(obj)s in bucket %(bucket)s") % {
'obj': obj,
'bucket': bucket
})
LOG.debug(msg)
raise exceptions.NotFound(message=msg)
return key
def delete(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file to delete
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises: NotFound if image does not exist
"""
loc = location.store_location
image = SheepdogImage(loc.addr, loc.port, loc.image,
self.WRITE_CHUNKSIZE)
if not image.exist():
raise exceptions.NotFound(
_("Sheepdog image %s does not exist") % loc.image)
image.delete()
def add(self,
image_id,
image_file,
image_size,
context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
full_data_path = self.path + "/" + image_id
LOG.debug(
_("connecting to %(host)s for %(data)s" % ({
'host': self.host,
'data': full_data_path
})))
bytes_written, checksum_hex = self.irods_manager.add_image_file(
full_data_path, image_file)
loc = StoreLocation(
{
'scheme': 'irods',
'host': self.host,
'port': self.port,
'zone': self.zone,
'path': self.path,
'user': self.user,
'password': self.password,
'data_name': image_id
}, None)
return (loc.get_uri(), bytes_written, checksum_hex, {})
def _load_config(self):
if self.backend_group:
scf = getattr(self.conf,
self.backend_group).swift_store_config_file
else:
scf = self.conf.glance_store.swift_store_config_file
try:
conf_file = self.conf.find_file(scf)
CONFIG.read(conf_file)
except Exception as e:
msg = (_("swift config file "
"%(conf)s:%(exc)s not found"), {
'conf': scf,
'exc': e
})
LOG.error(msg)
raise exceptions.BadStoreConfiguration(store_name='swift',
reason=msg)
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
for param in ('auth_address', 'user', 'key',
'project_domain_id', 'project_domain_name',
'user_domain_id', 'user_domain_name'):
reference[param] = CONFIG.get(ref, param)
try:
reference['auth_version'] = CONFIG.get(ref, 'auth_version')
except configparser.NoOptionError:
if self.backend_group:
av = getattr(
self.conf,
self.backend_group).swift_store_auth_version
else:
av = self.conf.glance_store.swift_store_auth_version
reference['auth_version'] = av
account_params[ref] = reference
except (ValueError, SyntaxError, configparser.NoOptionError):
LOG.exception(_LE("Invalid format of swift store config cfg"))
return account_params
def get_connection(self, conffile, rados_id):
client = rados.Rados(conffile=conffile, rados_id=rados_id)
try:
client.connect(timeout=self.connect_timeout)
except (rados.Error, rados.ObjectNotFound) as e:
if self.backend_group and len(self.conf.enabled_backends) > 1:
reason = _("Error in store configuration: %s") % e
LOG.debug(reason)
raise exceptions.BadStoreConfiguration(
store_name=self.backend_group, reason=reason)
else:
msg = _LE("Error connecting to ceph cluster.")
LOG.exception(msg)
raise exceptions.BackendException()
try:
yield client
finally:
client.shutdown()
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns the size
:param location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
"""
try:
size = self._query(location, 'HEAD')[2]
except socket.error:
reason = _("The HTTP URL is invalid.")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
except Exception:
# NOTE(flaper87): Catch more granular exceptions,
# keeping this branch for backwards compatibility.
return 0
return size
def _get_storage_url(self):
"""Get swift endpoint from keystone
Return endpoint for swift from service catalog. The method works only
Keystone v3. If you are using different version (1 or 2)
it returns None.
:return: swift endpoint
"""
if self.store.auth_version == '3':
try:
return self.client.session.get_endpoint(
service_type=self.store.service_type,
interface=self.store.endpoint_type,
region_name=self.store.region)
except Exception as e:
# do the same that swift driver does
# when catching ClientException
msg = _("Cannot find swift service endpoint : "
"%s") % encodeutils.exception_to_unicode(e)
raise exceptions.BackendException(msg)
def get_image_file_size(self, full_data_path):
"""
Returns image size for the file specified or returns 0
"""
try:
file_object = self.irods_conn_object.data_objects.get(
full_data_path)
except:
msg = _("image size %s not found") % full_data_path
Log.error(msg)
return 0
LOG.debug("path = %(path)s, size = %(data_size)s" %
({
'path': full_data_path,
'data_size': file_object.size
}))
return file_object.size
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns the size
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises: `glance_store.exceptions.NotFound` if image does not exist
"""
LOG.debug(
_("get_size() called with location: %s, context: %s" %
(location, context)))
try:
image = self._image_from_location(location.store_location)
return image.data_size
except Exception as e:
# Logging exceptions here because Glance has a tendancy to
# suppress them
LOG.error(e, exc_info=1)
raise
def configure_add(self):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
if pymongo is None:
msg = _("Missing dependencies: pymongo")
raise exceptions.BadStoreConfiguration(store_name="gridfs",
reason=msg)
self.mongodb_uri = self._option_get('mongodb_store_uri')
parsed = uri_parser.parse_uri(self.mongodb_uri)
self.mongodb_db = self._option_get('mongodb_store_db') or \
parsed.get("database")
self.mongodb = pymongo.MongoClient(self.mongodb_uri)
self.fs = gridfs.GridFS(self.mongodb[self.mongodb_db])
def set_acls(location_uri,
public=False,
read_tenants=[],
write_tenants=None,
context=None):
if write_tenants is None:
write_tenants = []
loc = location.get_location_from_uri(location_uri, conf=CONF)
scheme = get_store_from_location(location_uri)
store = get_store_from_scheme(scheme)
try:
store.set_acls(loc,
public=public,
read_tenants=read_tenants,
write_tenants=write_tenants,
context=context)
except NotImplementedError:
LOG.debug(_("Skipping store.set_acls... not implemented."))
def _query(self, location, method):
session = new_session(self.api_insecure, self.ca_file)
loc = location.store_location
redirects_followed = 0
# TODO(sabari): The redirect logic was added to handle cases when the
# backend redirects http url's to https. But the store never makes a
# http request and hence this can be safely removed.
while redirects_followed < MAX_REDIRECTS:
conn, resp = self._retry_request(session, method, location)
# NOTE(sigmavirus24): _retry_request handles 4xx and 5xx errors so
# if the response is not a redirect, we can return early.
if not conn.is_redirect:
break
redirects_followed += 1
location_header = conn.headers.get('location')
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.") % {
'path': loc.path,
'status': resp.status
})
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
conn.close()
location = self._new_location(location, location_header)
else:
# NOTE(sigmavirus24): We exceeded the maximum number of redirects
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.", {
'max_redirects': MAX_REDIRECTS
})
LOG.debug(msg)
raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def configure_add(self):
"""
Configure the Store to use the stored configuration options
Any store that needs special configuration should implement
this method. If the store was not able to successfully configure
itself, it should raise `exceptions.BadStoreConfiguration`
"""
try:
if self.backend_group:
chunk = getattr(self.conf,
self.backend_group).rbd_store_chunk_size
pool = getattr(self.conf, self.backend_group).rbd_store_pool
user = getattr(self.conf, self.backend_group).rbd_store_user
conf_file = getattr(self.conf,
self.backend_group).rbd_store_ceph_conf
connect_timeout = getattr(
self.conf, self.backend_group).rados_connect_timeout
else:
chunk = self.conf.glance_store.rbd_store_chunk_size
pool = self.conf.glance_store.rbd_store_pool
user = self.conf.glance_store.rbd_store_user
conf_file = self.conf.glance_store.rbd_store_ceph_conf
connect_timeout = self.conf.glance_store.rados_connect_timeout
self.chunk_size = chunk * units.Mi
self.READ_CHUNKSIZE = self.chunk_size
self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
# these must not be unicode since they will be passed to a
# non-unicode-aware C library
self.pool = str(pool)
self.user = str(user)
self.conf_file = str(conf_file)
self.connect_timeout = connect_timeout
except cfg.ConfigFileValueError as e:
reason = _("Error in store configuration: %s") % e
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name='rbd',
reason=reason)
if self.backend_group:
self._set_url_prefix()
def add(self, image_id, image_file, image_size, context=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance_store.exceptions.Duplicate` if the image already
existed
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.WRITE_CHUNKSIZE)
if image.exist():
raise exceptions.Duplicate(
_("Sheepdog image %s already exists") % image_id)
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image.create(image_size)
try:
total = left = image_size
while left > 0:
length = min(self.chunk_size, left)
data = image_file.read(length)
image.write(data, total - left, length)
left -= length
checksum.update(data)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exceptions.
with excutils.save_and_reraise_exception():
image.delete()
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def _v2_auth(self, token_url):
creds = self.creds
creds = {
"auth": {
"tenantName": creds['tenant'],
"passwordCredentials": {
"username": creds['username'],
"password": creds['password']
}
}
}
headers = {}
headers['Content-Type'] = 'application/json'
req_body = jsonutils.dumps(creds)
resp, resp_body = self._do_request(token_url,
'POST',
headers=headers,
body=req_body)
if resp.status == 200:
resp_auth = jsonutils.loads(resp_body)['access']
creds_region = self.creds.get('region')
if self.configure_via_auth:
endpoint = get_endpoint(resp_auth['serviceCatalog'],
endpoint_region=creds_region)
self.management_url = endpoint
self.auth_token = resp_auth['token']['id']
elif resp.status == 305:
raise exceptions.RedirectException(resp['location'])
elif resp.status == 400:
raise exceptions.AuthBadRequest(url=token_url)
elif resp.status == 401:
raise exceptions.NotAuthenticated()
elif resp.status == 404:
raise exceptions.AuthUrlNotFound(url=token_url)
else:
raise Exception(_('Unexpected response: %s') % resp.status)
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns the size
:param location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises `glance_store.exceptions.NotFound` if image does not exist
"""
loc = location.store_location
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
try:
with rbd.Image(ioctx, loc.image,
snapshot=loc.snapshot) as image:
img_info = image.stat()
return img_info['size']
except rbd.ImageNotFound:
msg = _('RBD image %s does not exist') % loc.get_uri()
LOG.debug(msg)
raise exceptions.NotFound(msg)
def configure(self, re_raise_bsc=False):
"""
Configure the store to use the stored configuration options
and initialize capabilities based on current configuration.
Any store that needs special configuration should implement
this method.
"""
try:
self.configure_add()
except exceptions.BadStoreConfiguration as e:
self.unset_capabilities(capabilities.BitMasks.WRITE_ACCESS)
msg = (_(u"Failed to configure store correctly: %s "
"Disabling add method.") %
encodeutils.exception_to_unicode(e))
LOG.warning(msg)
if re_raise_bsc:
raise
finally:
self.update_capabilities()
def get_image_file(self, full_data_path):
"""
Looks for the image on the path specified or raises exception
"""
try:
file_object = self.irods_conn_object.data_objects.get(
full_data_path)
except:
msg = _("image file %s not found") % full_data_path
Log.error(msg)
raise exceptions.NotFound(msg)
LOG.debug("path = %(path)s, size = %(data_size)s" %
({
'path': full_data_path,
'data_size': file_object.size
}))
return file_object, file_object.size
def delete(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file to delete
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:raises: NotFound if image does not exist
:raises: Forbidden if cannot delete because of permissions
"""
loc = location.store_location
self._check_context(context)
try:
volume = self.get_cinderclient(context).volumes.get(loc.volume_id)
volume.delete()
except cinder_exception.NotFound:
raise exceptions.NotFound(image=loc.volume_id)
except cinder_exception.ClientException as e:
msg = (_('Failed to delete volume %(volume_id)s: %(error)s') %
{'volume_id': loc.volume_id, 'error': e})
raise exceptions.BackendException(msg)
