def parse_uri(self, uri):
prefix = 'rbd://'
if not uri.startswith(prefix):
reason = _('URI must start with rbd://')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exception.BadStoreUri(message=reason)
# convert to ascii since librbd doesn't handle unicode
try:
ascii_uri = str(uri)
except UnicodeError:
reason = _('URI contains non-ascii characters')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exception.BadStoreUri(message=reason)
pieces = ascii_uri[len(prefix):].split('/')
if len(pieces) == 1:
self.fsid, self.pool, self.image, self.snapshot = \
(None, None, pieces[0], None)
elif len(pieces) == 4:
self.fsid, self.pool, self.image, self.snapshot = \
map(urlparse.unquote, pieces)
else:
reason = _('URI must have exactly 1 or 4 components')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exception.BadStoreUri(message=reason)
if any(map(lambda p: p == '', pieces)):
reason = _('URI cannot contain empty components')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exception.BadStoreUri(message=reason)
def _query(self, location, verb, depth=0):
if depth > MAX_REDIRECTS:
reason = (_("The HTTP URL exceeded %s maximum "
"redirects.") % MAX_REDIRECTS)
LOG.debug(reason)
raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
loc = location.store_location
conn_class = self._get_conn_class(loc)
conn = conn_class(loc.netloc)
conn.request(verb, loc.path, "", {})
resp = conn.getresponse()
# Check for bad status codes
if resp.status >= 400:
reason = _("HTTP URL returned a %s status code.") % resp.status
LOG.debug(reason)
raise exception.BadStoreUri(loc.path, reason)
location_header = resp.getheader("location")
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL attempted to redirect with an "
"invalid %s status code.") % resp.status)
LOG.debug(reason)
raise exception.BadStoreUri(loc.path, reason)
location_class = glance.store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, verb, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def get_connection(self, location):
if not location.user:
reason = (_("Location is missing user:password information."))
LOG.debug(reason)
raise exception.BadStoreUri(message=reason)
auth_url = location.swift_url
if not auth_url.endswith('/'):
auth_url += '/'
if self.auth_version == '2':
try:
tenant_name, user = location.user.split(':')
except ValueError:
reason = (_("Badly formed tenant:user '%(user)s' in "
"Swift URI") % {'user': location.user})
LOG.debug(reason)
raise exception.BadStoreUri()
else:
tenant_name = None
user = location.user
os_options = {}
if self.region:
os_options['region_name'] = self.region
os_options['endpoint_type'] = self.endpoint_type
os_options['service_type'] = self.service_type
return swiftclient.Connection(
auth_url, user, location.key, insecure=self.insecure,
tenant_name=tenant_name, snet=self.snet,
auth_version=self.auth_version, os_options=os_options)
def parse_uri(self, uri):
if not uri.startswith('%s://' % STORE_SCHEME):
reason = (_("URI %(uri)s must start with %(scheme)s://") % {
'uri': uri,
'scheme': STORE_SCHEME
})
LOG.error(reason)
raise exception.BadStoreUri(reason)
(self.scheme, self.server_host, path, params, query,
fragment) = urlparse.urlparse(uri)
if not query:
path = path.split('?')
if self._is_valid_path(path[0]):
self.path = path[0]
self.query = path[1]
return
elif self._is_valid_path(path):
self.path = path
self.query = query
return
reason = (_('Badly formed VMware datastore URI %(uri)s.') % {
'uri': uri
})
LOG.debug(reason)
raise exception.BadStoreUri(reason)
def validate_location_uri(location):
"""Validate location uri into acceptable format.
:param location: Location uri to be validated
"""
if not location:
raise exception.BadStoreUri(_('Invalid location: %s') % location)
elif location.startswith(('http://', 'https://')):
return location
# NOTE: file type uri is being avoided for security reasons,
# see LP bug #942118 #1400966.
elif location.startswith(("file:///", "filesystem:///")):
msg = _("File based imports are not allowed. Please use a non-local "
"source of image data.")
# NOTE: raise BadStoreUri and let the encompassing block save the error
# msg in the task.message.
raise exception.BadStoreUri(msg)
else:
# TODO(nikhil): add other supported uris
supported = [
'http',
]
msg = _("The given uri is not valid. Please specify a "
"valid uri from the following list of supported uri "
"%(supported)s") % {
'supported': supported
}
raise urllib.error.URLError(msg)
def parse_uri(self, uri):
valid_schema = 'sheepdog://'
if not uri.startswith(valid_schema):
raise exception.BadStoreUri(_("URI must start with %s://") %
valid_schema)
self.image = uri[len(valid_schema):]
if not utils.is_uuid_like(self.image):
raise exception.BadStoreUri(_("URI must contains well-formated "
"image id"))
def _query(self, location, method, depth=0):
if depth > MAX_REDIRECTS:
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.", {
'max_redirects': MAX_REDIRECTS
})
LOG.debug(msg)
raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
loc = location.store_location
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Cookie': cookie}
try:
conn = self._get_http_conn(method, loc, headers)
resp = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to access image %(image)s content.'),
{'image': location.image_id})
if resp.status >= 400:
if resp.status == httplib.UNAUTHORIZED:
self._create_session()
raise exception.NotAuthenticated()
if resp.status == httplib.NOT_FOUND:
msg = 'VMware datastore could not find image at URI.'
LOG.debug(msg)
raise exception.NotFound(msg)
reason = (_('HTTP request returned a %(status)s status code.') % {
'status': resp.status
})
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
location_header = resp.getheader('location')
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.") % {
'path': loc.path,
'status': resp.status
})
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
location_class = glance.store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, method, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def parse_uri(self, uri):
if not uri.startswith('cinder://'):
reason = _("URI must start with 'cinder://'")
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
self.scheme = 'cinder'
self.volume_id = uri[9:]
if not utils.is_uuid_like(self.volume_id):
reason = _("URI contains invalid volume ID")
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
def parse_uri(self, uri):
if not uri.startswith('cinder://'):
reason = _("URI must start with cinder://")
LOG.error(reason)
raise exception.BadStoreUri(uri, reason)
self.scheme = 'cinder'
self.volume_id = uri[9:]
if not uuidutils.is_uuid_like(self.volume_id):
reason = _("URI contains invalid volume ID: %s") % self.volume_id
LOG.error(reason)
raise exception.BadStoreUri(uri, reason)
def _form_uri_parts(self, netloc, path):
if netloc != '':
# > Python 2.6.1
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
else:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
netloc = path[0:path.find('/')].strip('/')
path = path[path.find('/'):].strip('/')
if creds:
cred_parts = creds.split(':')
if len(cred_parts) < 2:
reason = _("Badly formed credentials in Swift URI.")
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
key = cred_parts.pop()
user = '******'.join(cred_parts)
creds = urllib.unquote(creds)
try:
self.user, self.key = creds.rsplit(':', 1)
except exception.BadStoreConfiguration:
self.user = urllib.unquote(user)
self.key = urllib.unquote(key)
else:
self.user = None
self.key = None
return netloc, path
def _check_location_uri(context, store_api, store_utils, uri, backend=None):
"""Check if an image location is valid.
:param context: Glance request context
:param store_api: store API module
:param store_utils: store utils module
:param uri: location's uri string
:param backend: A backend name for the store
"""
try:
# NOTE(zhiyan): Some stores return zero when it catch exception
if CONF.enabled_backends:
size_from_backend = store_api.get_size_from_uri_and_backend(
uri, backend, context=context)
else:
size_from_backend = store_api.get_size_from_backend(
uri, context=context)
is_ok = (store_utils.validate_external_location(uri)
and size_from_backend > 0)
except (store.UnknownScheme, store.NotFound, store.BadStoreUri):
is_ok = False
if not is_ok:
reason = _('Invalid location')
raise exception.BadStoreUri(message=reason)
def set_attr(self, value):
if not isinstance(value, (list, StoreLocations)):
reason = _('Invalid locations')
raise exception.BadStoreUri(message=reason)
ori_value = getattr(getattr(self, target), attr)
if ori_value != value:
# NOTE(flwang): If all the URL of passed-in locations are same as
# current image locations, that means user would like to only
# update the metadata, not the URL.
ordered_value = sorted([loc['url'] for loc in value])
ordered_ori = sorted([loc['url'] for loc in ori_value])
if len(ori_value) > 0 and ordered_value != ordered_ori:
raise exception.Invalid(
_('Original locations is not empty: '
'%s') % ori_value)
# NOTE(zhiyan): Check locations are all valid
# NOTE(flwang): If all the URL of passed-in locations are same as
# current image locations, then it's not necessary to verify those
# locations again. Otherwise, if there is any restricted scheme in
# existing locations. _check_image_location will fail.
if ordered_value != ordered_ori:
for loc in value:
_check_image_location(self.context, self.store_api,
self.store_utils, loc)
loc['status'] = 'active'
if _count_duplicated_locations(value, loc) > 1:
raise exception.DuplicateLocation(location=loc['url'])
_set_image_size(self.context, getattr(self, target), value)
else:
for loc in value:
loc['status'] = 'active'
return setattr(getattr(self, target), attr, list(value))
def _make_swift_connection(self, auth_url, user, key):
"""
Creates a connection using the Swift client library.
"""
snet = self.snet
auth_version = self.auth_version
full_auth_url = (auth_url if not auth_url or auth_url.endswith('/')
else auth_url + '/')
logger.debug(
_("Creating Swift connection with "
"(auth_address=%(full_auth_url)s, user=%(user)s, "
"snet=%(snet)s, auth_version=%(auth_version)s)") % locals())
tenant_name = None
if self.auth_version == '2':
tenant_user = user.split(':')
if len(tenant_user) != 2:
reason = (_("Badly formed tenant:user '%(tenant_user)s' in "
"Swift URI") % locals())
raise exception.BadStoreUri(auth_url, reason)
(tenant_name, user) = tenant_user
return swift_client.Connection(authurl=full_auth_url,
user=user,
key=key,
snet=snet,
tenant_name=tenant_name,
auth_version=auth_version)
def _make_swift_connection(self,
auth_url,
user,
key,
region=None,
storage_url=None,
token=None):
"""
Creates a connection using the Swift client library.
:param auth_url The authentication for v1 style Swift auth or
v2 style Keystone auth.
:param user A string containing the tenant:user information.
:param key A string containing the key/password for the connection.
:param region A string containing the swift endpoint region
:param storage_url A string containing the storage URL.
:param token A string containing the token
"""
snet = self.snet
auth_version = self.auth_version
full_auth_url = (auth_url if not auth_url or auth_url.endswith('/')
else auth_url + '/')
LOG.debug(
_("Creating Swift connection with "
"(auth_address=%(full_auth_url)s, user=%(user)s, "
"snet=%(snet)s, auth_version=%(auth_version)s)") % locals())
tenant_name = None
if self.auth_version == '2':
tenant_user = user.split(':')
if len(tenant_user) != 2:
reason = (_("Badly formed tenant:user '%(tenant_user)s' in "
"Swift URI") % locals())
LOG.error(reason)
raise exception.BadStoreUri()
(tenant_name, user) = tenant_user
if self.multi_tenant:
#NOTE: multi-tenant supports v2 auth only
return swiftclient.Connection(None,
user,
None,
preauthurl=storage_url,
preauthtoken=token,
snet=snet,
tenant_name=tenant_name,
auth_version='2')
else:
os_options = {}
if region:
os_options['region_name'] = region
return swiftclient.Connection(full_auth_url,
user,
key,
snet=snet,
os_options=os_options,
tenant_name=tenant_name,
auth_version=auth_version)
def _query(self, location, method, headers, depth=0):
if depth > MAX_REDIRECTS:
msg = (_("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.") % {
'max_redirects': MAX_REDIRECTS
})
LOG.debug(msg)
raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
loc = location.store_location
conn = self._get_http_conn(method, loc, headers)
resp = conn.getresponse()
if resp.status >= 400:
if resp.status == httplib.NOT_FOUND:
msg = _('VMware datastore could not find image at URI.')
LOG.debug(msg)
raise exception.NotFound(msg)
msg = (_('HTTP URL %(url)s returned a %(status)s status code.') % {
'url': loc.get_uri(),
'status': resp.status
})
LOG.debug(msg)
raise exception.BadStoreUri(msg)
location_header = resp.getheader('location')
if location_header:
if resp.status not in (301, 302):
msg = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.") % {
'path': loc.path,
'status': resp.status
})
LOG.debug(msg)
raise exception.BadStoreUri(msg)
location_class = glance.store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, method, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def parse_uri(self, uri):
if not uri.startswith('%s://' % STORE_SCHEME):
reason = (_("URI must start with %s://") % STORE_SCHEME)
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
(self.scheme, self.server_host, path, params, query,
fragment) = urlparse.urlparse(uri)
if not query:
path = path.split('?')
if self._is_valid_path(path[0]):
self.path = path[0]
self.query = path[1]
return
elif self._is_valid_path(path):
self.path = path
self.query = query
return
reason = _('Badly formed VMware datastore URI')
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
def parse_uri(self, uri):
prefix = 'rbd://'
if not uri.startswith(prefix):
reason = _('URI must start with rbd://')
LOG.error(_("Invalid URI: %(uri)s: %(reason)s") % locals())
raise exception.BadStoreUri(message=reason)
pieces = uri[len(prefix):].split('/')
if len(pieces) == 1:
self.fsid, self.pool, self.image, self.snapshot = \
(None, None, pieces[0], None)
elif len(pieces) == 4:
self.fsid, self.pool, self.image, self.snapshot = \
map(urllib.unquote, pieces)
else:
reason = _('URI must have exactly 1 or 4 components')
LOG.error(_("Invalid URI: %(uri)s: %(reason)s") % locals())
raise exception.BadStoreUri(message=reason)
if any(map(lambda p: p == '', pieces)):
reason = _('URI cannot contain empty components')
LOG.error(_("Invalid URI: %(uri)s: %(reason)s") % locals())
raise exception.BadStoreUri(message=reason)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('https', 'http')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path
try:
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
except ValueError:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
if creds:
try:
self.user, self.password = creds.split(':')
except ValueError:
reason = (_("Credentials '%s' not well-formatted.") %
"".join(creds))
LOG.debug(reason)
raise exception.BadStoreUri()
else:
self.user = None
if netloc == '':
reason = _("No address specified in HTTP URL")
LOG.debug(reason)
raise exception.BadStoreUri(message=reason)
self.netloc = netloc
self.path = path
def _swift_connection_for_location(self, loc):
if loc.user:
return self._make_swift_connection(
loc.swift_url, loc.user, loc.key, region=self.region)
else:
if self.multi_tenant:
return self._make_swift_connection(
None, self.user, None,
storage_url=loc.swift_url, token=self.token)
else:
reason = (_("Location is missing user:password information."))
LOG.error(reason)
raise exception.BadStoreUri(message=reason)
def _form_auth_or_store_url(self, netloc, path):
path_parts = path.split('/')
try:
self.obj = path_parts.pop()
self.container = path_parts.pop()
if not netloc.startswith('http'):
# push hostname back into the remaining to build full authurl
path_parts.insert(0, netloc)
self.auth_or_store_url = '/'.join(path_parts)
except IndexError:
reason = _("Badly formed Swift URI.")
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('file', 'filesystem')
self.scheme = pieces.scheme
path = (pieces.netloc + pieces.path).strip()
if path == '':
reason = _("No path specified")
raise exception.BadStoreUri(uri, reason)
self.path = path
def set_attr(self, value):
if not isinstance(value, (list, StoreLocations)):
raise exception.BadStoreUri(_('Invalid locations: %s') % value)
ori_value = getattr(getattr(self, target), attr)
if ori_value != value:
# NOTE(zhiyan): Enforced locations list was previously empty list.
if len(ori_value) > 0:
raise exception.Invalid(
_('Original locations is not empty: '
'%s') % ori_value)
# NOTE(zhiyan): Check locations are all valid.
for location in value:
_check_image_location(self.context, self.store_api, location)
return setattr(getattr(self, target), attr, list(value))
def _build_staging_store():
conf = cfg.ConfigOpts()
backend.register_opts(conf)
conf.set_override('filesystem_store_datadir',
CONF.node_staging_uri[7:],
group='glance_store')
staging_store = backend._load_store(conf, 'file')
try:
staging_store.configure()
except AttributeError:
msg = _("'node_staging_uri' is not set correctly. Could not "
"load staging store.")
raise exception.BadStoreUri(message=msg)
return staging_store
def _check_location_uri(context, store_api, uri):
"""
Check if an image location uri is valid.
:param context: Glance request context
:param store_api: store API module
:param uri: location's uri string
"""
is_ok = True
try:
size = store_api.get_size_from_backend(context, uri)
# NOTE(zhiyan): Some stores return zero when it catch exception
is_ok = size > 0
except (exception.UnknownScheme, exception.NotFound):
is_ok = False
if not is_ok:
raise exception.BadStoreUri(_('Invalid location: %s') % uri)
def parse_uri(self, uri):
(self.scheme, self.server_host,
path, params, query, fragment) = urlparse.urlparse(uri)
if not query:
path = path.split('?')
if self._is_valid_path(path[0]):
self.path = path[0]
self.query = path[1]
return
elif self._is_valid_path(path):
self.path = path
self.query = query
return
reason = (_('Badly formed VMware datastore URI %(uri)s.')
% {'uri': uri})
LOG.debug(reason)
raise exception.BadStoreUri(reason)
def _get_conf_value_from_account_ref(self, netloc):
try:
self.user = SWIFT_STORE_REF_PARAMS[netloc]['user']
self.key = SWIFT_STORE_REF_PARAMS[netloc]['key']
netloc = SWIFT_STORE_REF_PARAMS[netloc]['auth_address']
self.ssl_enabled = True
if netloc != '':
if netloc.startswith('http://'):
self.ssl_enabled = False
netloc = netloc[len('http://'):]
elif netloc.startswith('https://'):
netloc = netloc[len('https://'):]
except KeyError:
reason = _("Badly formed Swift URI. Credentials not found for"
"account reference")
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
return netloc
def set_attr(self, value):
if not isinstance(value, (list, StoreLocations)):
reason = _('Invalid locations')
raise exception.BadStoreUri(message=reason)
ori_value = getattr(getattr(self, target), attr)
if ori_value != value:
# NOTE(zhiyan): Enforced locations list was previously empty list.
if len(ori_value) > 0:
raise exception.Invalid(
_('Original locations is not empty: '
'%s') % ori_value)
# NOTE(zhiyan): Check locations are all valid.
for location in value:
_check_image_location(self.context, self.store_api, location)
location['status'] = 'active'
if _count_duplicated_locations(value, location) > 1:
raise exception.DuplicateLocation(location=location['url'])
_set_image_size(self.context, getattr(self, target), value)
return setattr(getattr(self, target), attr, list(value))
def _check_location_uri(context, store_api, store_utils, uri):
"""Check if an image location is valid.
:param context: Glance request context
:param store_api: store API module
:param store_utils: store utils module
:param uri: location's uri string
"""
is_ok = True
try:
# NOTE(zhiyan): Some stores return zero when it catch exception
is_ok = (store_utils.validate_external_location(uri)
and store_api.get_size_from_backend(uri, context=context) > 0)
except (store.UnknownScheme, store.NotFound):
is_ok = False
if not is_ok:
reason = _('Invalid location')
raise exception.BadStoreUri(message=reason)
def get_size(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns the size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
"""
try:
size = self._query(location, 'HEAD')[2]
except socket.error:
reason = _("The HTTP URL is invalid.")
LOG.debug(reason)
raise exception.BadStoreUri(reason)
except Exception:
# NOTE(flaper87): Catch more granular exceptions,
# keeping this branch for backwards compatibility.
return 0
return size
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python. It also deals with the peculiarity that new-style
Swift URIs have where a username can contain a ':', like so:
swift://account:user:[email protected]/container/obj
and for system created locations with account reference
swift+config://account_reference/container/obj
"""
# Make sure that URIs that contain multiple schemes, such as:
# swift://user:pass@http://authurl.com/v1/container/obj
# are immediately rejected.
if uri.count('://') != 1:
reason = _("URI cannot contain more than one occurrence "
"of a scheme. If you have specified a URI like "
"swift://*****:*****@http://authurl.com/v1/container/obj"
", you need to change it to use the "
"swift+http:// scheme, like so: "
"swift+http://user:[email protected]/v1/container/obj")
LOG.info(_LI("Invalid store URI: %(reason)s"), {'reason': reason})
raise exception.BadStoreUri(message=reason)
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https',
'swift+config')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path.lstrip('/')
# NOTE(Sridevi): Fix to map the account reference to the
# corresponding CONF value
if self.scheme == 'swift+config':
netloc = self._get_conf_value_from_account_ref(netloc)
else:
netloc, path = self._form_uri_parts(netloc, path)
self._form_auth_or_store_url(netloc, path)
