def read(self, length=None):
"""Return the requested amount of bytes, fetching the next chunk of
the underlying iterator when needed.
This is replaced with cooperative_read in __init__ if the underlying
fd already supports read().
"""
if length is None:
if len(self.buffer) - self.position > 0:
# if no length specified but some data exists in buffer,
# return that data and clear the buffer
result = self.buffer[self.position:]
self.buffer = b''
self.position = 0
return str(result)
else:
# otherwise read the next chunk from the underlying iterator
# and return it as a whole. Reset the buffer, as subsequent
# calls may specify the length
try:
if self.iterator is None:
self.iterator = self.__iter__()
return next(self.iterator)
except StopIteration:
return ''
finally:
self.buffer = b''
self.position = 0
else:
result = bytearray()
while len(result) < length:
if self.position < len(self.buffer):
to_read = length - len(result)
chunk = self.buffer[self.position:self.position + to_read]
result.extend(chunk)
# This check is here to prevent potential OOM issues if
# this code is called with unreasonably high values of read
# size. Currently it is only called from the HTTP clients
# of Glance backend stores, which use httplib for data
# streaming, which has readsize hardcoded to 8K, so this
# check should never fire. Regardless it still worths to
# make the check, as the code may be reused somewhere else.
if len(result) >= MAX_COOP_READER_BUFFER_SIZE:
raise exception.LimitExceeded()
self.position += len(chunk)
else:
try:
if self.iterator is None:
self.iterator = self.__iter__()
self.buffer = next(self.iterator)
self.position = 0
except StopIteration:
self.buffer = b''
self.position = 0
return bytes(result)
return bytes(result)
def _enforce_some(context, project_id, quota_value_fns, deltas):
"""Helper method to enforce a set of quota values.
:param context: The RequestContext
:param project_id: The project_id of the tenant being checked
:param get_value_fns: A mapping of quota names to functions that will be
called with no arguments to return the numerical
value representing current usage.
:param deltas: A mapping of quota names to the amount of resource being
requested for each (to be added to the current usage before
determining if over-quota).
:raises: exception.LimitExceeded if the current usage is over the defined
limit.
:returns: None if the tenant is not currently over their quota.
"""
if not CONF.use_keystone_limits:
return
def callback(project_id, resource_names):
return {name: quota_value_fns[name]() for name in resource_names}
enforcer = limit.Enforcer(callback)
try:
enforcer.enforce(
project_id, {
quota_name: deltas.get(quota_name, 0)
for quota_name in quota_value_fns
})
except ol_exc.ProjectOverLimit as e:
raise exception.LimitExceeded(body=str(e))
except ol_exc.SessionInitError as e:
LOG.error(
_LE('Failed to initialize oslo_limit, likely due to '
'incorrect or insufficient configuration: %(err)s'),
{'err': str(e)})
# We could just raise LimitExceeded here, but a 500 is
# appropriate for incorrect server-side configuration, so we
# re-raise here after the above error message to make sure we
# are noticed.
raise
def _do_request(self, method, url, body, headers):
"""
Connects to the server and issues a request. Handles converting
any returned HTTP error status codes to OpenStack/Glance exceptions
and closing the server connection. Returns the result data, or
raises an appropriate exception.
:param method: HTTP method ("GET", "POST", "PUT", etc...)
:param url: urlparse.ParsedResult object with URL information
:param body: data to send (as string, filelike or iterable),
or None (default)
:param headers: mapping of key/value pairs to add as headers
:note
If the body param has a read attribute, and method is either
POST or PUT, this method will automatically conduct a chunked-transfer
encoding and use the body as a file object or iterable, transferring
chunks of data using the connection's send() method. This allows large
objects to be transferred efficiently without buffering the entire
body in memory.
"""
if url.query:
path = url.path + "?" + url.query
else:
path = url.path
try:
connection_type = self.get_connection_type()
headers = self._encode_headers(headers or {})
if 'x-auth-token' not in headers and self.auth_tok:
headers['x-auth-token'] = self.auth_tok
c = connection_type(url.hostname, url.port, **self.connect_kwargs)
def _pushing(method):
return method.lower() in ('post', 'put')
def _simple(body):
return body is None or isinstance(body, six.string_types)
def _filelike(body):
return hasattr(body, 'read')
def _sendbody(connection, iter):
connection.endheaders()
for sent in iter:
# iterator has done the heavy lifting
pass
def _chunkbody(connection, iter):
connection.putheader('Transfer-Encoding', 'chunked')
connection.endheaders()
for chunk in iter:
connection.send('%x\r\n%s\r\n' % (len(chunk), chunk))
connection.send('0\r\n\r\n')
# Do a simple request or a chunked request, depending
# on whether the body param is file-like or iterable and
# the method is PUT or POST
#
if not _pushing(method) or _simple(body):
# Simple request...
c.request(method, path, body, headers)
elif _filelike(body) or self._iterable(body):
c.putrequest(method, path)
use_sendfile = self._sendable(body)
# According to HTTP/1.1, Content-Length and Transfer-Encoding
# conflict.
for header, value in headers.items():
if use_sendfile or header.lower() != 'content-length':
c.putheader(header, str(value))
iter = self.image_iterator(c, headers, body)
if use_sendfile:
# send actual file without copying into userspace
_sendbody(c, iter)
else:
# otherwise iterate and chunk
_chunkbody(c, iter)
else:
raise TypeError('Unsupported image type: %s' % body.__class__)
res = c.getresponse()
def _retry(res):
return res.getheader('Retry-After')
status_code = self.get_status_code(res)
if status_code in self.OK_RESPONSE_CODES:
return res
elif status_code in self.REDIRECT_RESPONSE_CODES:
raise exception.RedirectException(res.getheader('Location'))
elif status_code == httplib.UNAUTHORIZED:
raise exception.NotAuthenticated(res.read())
elif status_code == httplib.FORBIDDEN:
raise exception.Forbidden(res.read())
elif status_code == httplib.NOT_FOUND:
raise exception.NotFound(res.read())
elif status_code == httplib.CONFLICT:
raise exception.Duplicate(res.read())
elif status_code == httplib.BAD_REQUEST:
raise exception.Invalid(res.read())
elif status_code == httplib.MULTIPLE_CHOICES:
raise exception.MultipleChoices(body=res.read())
elif status_code == httplib.REQUEST_ENTITY_TOO_LARGE:
raise exception.LimitExceeded(retry=_retry(res),
body=res.read())
elif status_code == httplib.INTERNAL_SERVER_ERROR:
raise exception.ServerError()
elif status_code == httplib.SERVICE_UNAVAILABLE:
raise exception.ServiceUnavailable(retry=_retry(res))
else:
raise exception.UnexpectedStatus(status=status_code,
body=res.read())
except (socket.error, IOError) as e:
raise exception.ClientConnectionError(e)
