def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
"""
pieces = urllib.parse.urlparse(uri)
assert pieces.scheme in ('file', 'filesystem')
self.scheme = pieces.scheme
path = (pieces.netloc + pieces.path).strip()
if path == '':
reason = _("No path specified in URI")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
self.path = path
def parse_uri(self, uri):
prefix = 'rbd://'
if not uri.startswith(prefix):
reason = _('URI must start with rbd://')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
# convert to ascii since librbd doesn't handle unicode
try:
ascii_uri = str(uri)
except UnicodeError:
reason = _('URI contains non-ascii characters')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
pieces = ascii_uri[len(prefix):].split('/')
if len(pieces) == 1:
self.fsid, self.pool, self.image, self.snapshot = \
(None, None, pieces[0], None)
elif len(pieces) == 4:
self.fsid, self.pool, self.image, self.snapshot = \
map(urllib.parse.unquote, pieces)
else:
reason = _('URI must have exactly 1 or 4 components')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
if any(map(lambda p: p == '', pieces)):
reason = _('URI cannot contain empty components')
msg = _LI("Invalid URI: %s") % reason
LOG.info(msg)
raise exceptions.BadStoreUri(message=reason)
def parse_uri(self, uri):
if not uri.startswith('%s://' % STORE_SCHEME):
reason = (_("URI %(uri)s must start with %(scheme)s://") % {
'uri': uri,
'scheme': STORE_SCHEME
})
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
(self.scheme, self.server_host, path, params, query,
fragment) = urlparse.urlparse(uri)
if not query:
path, query = path.split('?')
self.path = path
self.query = query
def parse_uri(self, uri):
valid_schema = 'sheepdog://'
if not uri.startswith(valid_schema):
reason = _("URI must start with '%s'") % valid_schema
raise exceptions.BadStoreUri(message=reason)
pieces = uri[len(valid_schema):].split(':')
if len(pieces) == 3:
self.image = pieces[2]
self.port = int(pieces[1])
self.addr = pieces[0]
# This is used for backwards compatibility.
else:
self.image = pieces[0]
self.port = self.conf.glance_store.sheepdog_store_port
self.addr = self.conf.glance_store.sheepdog_store_address
def _get_conf_value_from_account_ref(self, netloc):
try:
self.user = SWIFT_STORE_REF_PARAMS[netloc]['user']
self.key = SWIFT_STORE_REF_PARAMS[netloc]['key']
netloc = SWIFT_STORE_REF_PARAMS[netloc]['auth_address']
self.ssl_enabled = True
if netloc != '':
if netloc.startswith('http://'):
self.ssl_enabled = False
netloc = netloc[len('http://'):]
elif netloc.startswith('https://'):
netloc = netloc[len('https://'):]
except KeyError:
reason = _("Badly formed Swift URI. Credentials not found for "
"account reference")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
return netloc
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns the size
:param location `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
"""
try:
size = self._query(location, 'HEAD')[2]
except socket.error:
reason = _("The HTTP URL is invalid.")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
except Exception:
# NOTE(flaper87): Catch more granular exceptions,
# keeping this branch for backwards compatibility.
return 0
return size
def _query(self, location, method):
session = new_session(self.api_insecure, self.ca_file)
loc = location.store_location
redirects_followed = 0
# TODO(sabari): The redirect logic was added to handle cases when the
# backend redirects http url's to https. But the store never makes a
# http request and hence this can be safely removed.
while redirects_followed < MAX_REDIRECTS:
conn, resp = self._retry_request(session, method, location)
# NOTE(sigmavirus24): _retry_request handles 4xx and 5xx errors so
# if the response is not a redirect, we can return early.
if not conn.is_redirect:
break
redirects_followed += 1
location_header = conn.headers.get('location')
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.") % {
'path': loc.path,
'status': resp.status
})
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
conn.close()
location = self._new_location(location, location_header)
else:
# NOTE(sigmavirus24): We exceeded the maximum number of redirects
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.", {
'max_redirects': MAX_REDIRECTS
})
LOG.debug(msg)
raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python. It also deals with the peculiarity that new-style
Swift URIs have where a username can contain a ':', like so:
swift://account:user:[email protected]/container/obj
and for system created locations with account reference
swift+config://account_reference/container/obj
"""
# Make sure that URIs that contain multiple schemes, such as:
# swift://user:pass@http://authurl.com/v1/container/obj
# are immediately rejected.
if uri.count('://') != 1:
reason = _("URI cannot contain more than one occurrence "
"of a scheme. If you have specified a URI like "
"swift://*****:*****@http://authurl.com/v1/container/obj"
", you need to change it to use the "
"swift+http:// scheme, like so: "
"swift+http://user:[email protected]/v1/container/obj")
LOG.info(_LI("Invalid store URI: %(reason)s"), {'reason': reason})
raise exceptions.BadStoreUri(message=reason)
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https',
'swift+config')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path.lstrip('/')
# NOTE(Sridevi): Fix to map the account reference to the
# corresponding configuration value
if self.scheme == 'swift+config':
netloc = self._get_conf_value_from_account_ref(netloc)
else:
netloc, path = self._form_uri_parts(netloc, path)
self._form_auth_or_store_url(netloc, path)
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
where to find the image file, and returns the size
:param location: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
"""
conn = None
try:
conn, resp, size = self._query(location, 'HEAD')
except requests.exceptions.ConnectionError as exc:
err_msg = encodeutils.exception_to_unicode(exc)
reason = _("The HTTP URL is invalid: %s") % err_msg
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
finally:
# NOTE(sabari): Close the connection as the request was made with
# stream=True
if conn is not None:
conn.close()
return size
def _retry_request(self, session, method, location):
loc = location.store_location
# NOTE(arnaud): use a decorator when the config is not tied to self
for i in range(self.api_retry_count + 1):
cookie = self._build_vim_cookie_header()
headers = {'Cookie': cookie}
conn = session.request(method, loc.https_url, headers=headers,
stream=True)
resp = conn.raw
if resp.status >= 400:
if resp.status == requests.codes.unauthorized:
self.reset_session()
continue
if resp.status == requests.codes.not_found:
reason = _('VMware datastore could not find image at URI.')
LOG.info(reason)
raise exceptions.NotFound(message=reason)
msg = ('HTTP request returned a %(status)s status code.'
% {'status': resp.status})
LOG.debug(msg)
raise exceptions.BadStoreUri(msg)
break
return conn, resp
def parse_uri(self, uri):
if not uri.startswith('%s://' % STORE_SCHEME):
reason = (_("URI %(uri)s must start with %(scheme)s://") %
{'uri': uri, 'scheme': STORE_SCHEME})
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
(self.scheme, self.server_host,
path, params, query, fragment) = urllib.parse.urlparse(uri)
if not query:
path, query = path.split('?')
self.path = path
self.query = query
# NOTE(flaper87): Read comment on `_is_valid_path`
# reason = 'Badly formed VMware datastore URI %(uri)s.' % {'uri': uri}
# LOG.debug(reason)
# raise exceptions.BadStoreUri(reason)
parts = urllib.parse.parse_qs(self.query)
dc_path = parts.get('dcPath')
if dc_path:
self.datacenter_path = dc_path[0]
ds_name = parts.get('dsName')
if ds_name:
self.datastore_name = ds_name[0]
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python.
Note that an Amazon AWS secret key can contain the forward slash,
which is entirely retarded, and breaks urlparse miserably.
This function works around that issue.
"""
# Make sure that URIs that contain multiple schemes, such as:
# s3://accesskey:secretkey@https://s3.amazonaws.com/bucket/key-id
# are immediately rejected.
if uri.count('://') != 1:
reason = _("URI cannot contain more than one occurrence "
"of a scheme. If you have specified a URI like "
"s3://accesskey:secretkey@"
"https://s3.amazonaws.com/bucket/key-id"
", you need to change it to use the "
"s3+https:// scheme, like so: "
"s3+https://accesskey:secretkey@"
"s3.amazonaws.com/bucket/key-id")
LOG.info(_LI("Invalid store uri: %s") % reason)
raise exceptions.BadStoreUri(message=reason)
pieces = urllib.parse.urlparse(uri)
assert pieces.scheme in ('s3', 's3+http', 's3+https')
self.scheme = pieces.scheme
path = pieces.path.strip('/')
netloc = pieces.netloc.strip('/')
entire_path = (netloc + '/' + path).strip('/')
if '@' in uri:
creds, path = entire_path.split('@')
cred_parts = creds.split(':')
try:
access_key = cred_parts[0]
secret_key = cred_parts[1]
if six.PY2:
# NOTE(jaypipes): Need to encode to UTF-8 here because of a
# bug in the HMAC library that boto uses.
# See: http://bugs.python.org/issue5285
# See: http://trac.edgewall.org/ticket/8083
access_key = access_key.encode('utf-8')
secret_key = secret_key.encode('utf-8')
self.accesskey = access_key
self.secretkey = secret_key
except IndexError:
reason = _("Badly formed S3 credentials")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
else:
self.accesskey = None
path = entire_path
try:
path_parts = path.split('/')
self.key = path_parts.pop()
self.bucket = path_parts.pop()
if path_parts:
self.s3serviceurl = '/'.join(path_parts).strip('/')
else:
reason = _("Badly formed S3 URI. Missing s3 service URL.")
raise exceptions.BadStoreUri(message=reason)
except IndexError:
reason = _("Badly formed S3 URI")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
def validate_location(self, uri):
pieces = urlparse.urlparse(uri)
if pieces.scheme in ['swift+config']:
reason = (_("Location credentials are invalid"))
raise exceptions.BadStoreUri(message=reason)
def parse_uri(self, uri):
valid_schema = 'sheepdog://'
if not uri.startswith(valid_schema):
reason = _("URI must start with '%s://'") % valid_schema
raise exceptions.BadStoreUri(message=reason)
self.image = uri[11:]