def _enforce_image_property_quota(self,
image_meta,
orig_image_meta=None,
purge_props=False,
req=None):
if CONF.image_property_quota < 0:
# If value is negative, allow unlimited number of properties
return
props = image_meta['properties'].keys()
# NOTE(ameade): If we are not removing existing properties,
# take them in to account
if (not purge_props) and orig_image_meta:
original_props = orig_image_meta['properties'].keys()
props.extend(original_props)
props = set(props)
if len(props) > CONF.image_property_quota:
msg = (_("The limit has been exceeded on the number of allowed "
"image properties. Attempted: %s, Maximum: %s") %
(len(props), CONF.image_property_quota))
LOG.info(msg)
raise HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type="text/plain")
def check_object_creation(req, object_name):
"""
Check to ensure that everything is alright about an object to be created.
:param req: HTTP request object
:param object_name: name of object to be created
:raises HTTPRequestEntityTooLarge: the object is too large
:raises HTTPLengthRequered: missing content-length header and not
a chunked request
:raises HTTPBadRequest: missing or bad content-type header, or
bad metadata
"""
if req.content_length and req.content_length > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(body='Your request is too large.',
request=req,
content_type='text/plain')
if req.content_length is None and \
req.headers.get('transfer-encoding') != 'chunked':
return HTTPLengthRequired(request=req)
if 'X-Copy-From' in req.headers and req.content_length:
return HTTPBadRequest(body='Copy requests require a zero byte body',
request=req,
content_type='text/plain')
if len(object_name) > MAX_OBJECT_NAME_LENGTH:
return HTTPBadRequest(body='Object name length of %d longer than %d' %
(len(object_name), MAX_OBJECT_NAME_LENGTH),
request=req,
content_type='text/plain')
if 'Content-Type' not in req.headers:
return HTTPBadRequest(request=req,
content_type='text/plain',
body='No content type')
if not check_utf8(req.headers['Content-Type']):
return HTTPBadRequest(request=req,
body='Invalid Content-Type',
content_type='text/plain')
if 'x-object-manifest' in req.headers:
value = req.headers['x-object-manifest']
container = prefix = None
try:
container, prefix = value.split('/', 1)
except ValueError:
pass
if not container or not prefix or '?' in value or '&' in value or \
prefix[0] == '/':
return HTTPBadRequest(
request=req,
body='X-Object-Manifest must in the format container/prefix')
return check_metadata(req, 'object')
def make_service_call(self, req, env, method_dict, variables):
try:
req_method = req.method.upper()
call_pattern = method_dict[req_method]
call_pattern = self.replace_variables(req, call_pattern, variables)
self.logger.debug("Call on resource is: %s" % str(call_pattern))
response = eval(call_pattern)
except BadRequestException as error:
content_type, detail = self.get_error_body(req, error)
return HTTPBadRequest(body=detail,
content_type=content_type,
request=req)
except NotImplementedException as error:
content_type, detail = self.get_error_body(req, error)
return HTTPNotImplemented(body=detail,
content_type=content_type,
request=req)
except ServiceUnavailableException as error:
content_type, detail = self.get_error_body(req, error)
return HTTPInternalServerError(body=detail,
content_type=content_type,
request=req)
except ItemNotFoundException as error:
content_type, detail = self.get_error_body(req, error)
return HTTPNotFound(body=detail,
content_type=content_type,
request=req)
except OverLimitException as error:
content_type, detail = self.get_error_body(req, error)
return HTTPRequestEntityTooLarge(body=detail,
content_type=content_type,
request=req)
except ImmutableEntityException as error:
content_type, detail = self.get_error_body(req, error)
return HTTPUnprocessableEntity(body=detail,
content_type=content_type,
request=req)
except LoadBalancerFaultException as error:
content_type, detail = self.get_error_body(req, error)
return HTTPUnauthorized(body=detail,
content_type=content_type,
request=req)
except Exception, msg:
exc_type, exc_value, exc_tb = sys.exc_info()
tb = traceback.format_exception(exc_type, exc_value, exc_tb)
tb.reverse()
tb_str = ''.join(tb)
self.logger.debug("cannot eval(%s) because %s. Traceback: %s" %
(call_pattern, msg, tb_str))
error = ServiceUnavailableException(
"The Load Balancing Service is currently unavailable." +
" Please contact support")
content_type, detail = self.get_error_body(req, error)
return HTTPInternalServerError(body=detail,
content_type=content_type,
request=req)
def _upload(self, req, image_meta):
"""
Uploads the payload of the request to a backend store in
Glance. If the `x-image-meta-store` header is set, Glance
will attempt to use that scheme; if not, Glance will use the
scheme set by the flag `default_store` to find the backing store.
:param req: The WSGI/Webob Request object
:param image_meta: Mapping of metadata about image
:raises HTTPConflict if image already exists
:retval The location where the image was stored
"""
copy_from = self._copy_from(req)
if copy_from:
try:
image_data, image_size = self._get_from_store(req.context,
copy_from)
except Exception as e:
self._safe_kill(req, image_meta['id'])
msg = _("Copy from external source failed: %s") % e
LOG.debug(msg)
return
image_meta['size'] = image_size or image_meta['size']
else:
try:
req.get_content_type('application/octet-stream')
except exception.InvalidContentType:
self._safe_kill(req, image_meta['id'])
msg = _("Content-Type must be application/octet-stream")
LOG.debug(msg)
raise HTTPBadRequest(explanation=msg)
image_data = req.body_file
scheme = req.headers.get('x-image-meta-store', CONF.default_store)
store = self.get_store_or_400(req, scheme)
image_id = image_meta['id']
LOG.debug(_("Setting image %s to status 'saving'"), image_id)
registry.update_image_metadata(req.context, image_id,
{'status': 'saving'})
LOG.debug(_("Uploading image data for image %(image_id)s "
"to %(scheme)s store"), locals())
try:
self.notifier.info("image.prepare", redact_loc(image_meta))
location, size, checksum = store.add(
image_meta['id'],
utils.CooperativeReader(image_data),
image_meta['size'])
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = _("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % locals()
LOG.error(msg)
self._safe_kill(req, image_id)
self._initiate_deletion(req, location, image_id)
raise HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), locals())
update_data = {'checksum': checksum,
'size': size}
image_meta = registry.update_image_metadata(req.context,
image_id,
update_data)
self.notifier.info('image.upload', redact_loc(image_meta))
return location
except exception.Duplicate as e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.debug(msg)
self._safe_kill(req, image_id)
raise HTTPConflict(explanation=msg, request=req)
except exception.Forbidden as e:
msg = _("Forbidden upload attempt: %s") % e
LOG.debug(msg)
self._safe_kill(req, image_id)
raise HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPRequestEntityTooLarge(explanation=msg, request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPServiceUnavailable(explanation=msg, request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = _("Denying attempt to upload image larger than %d bytes."
% CONF.image_size_cap)
LOG.info(msg)
self._safe_kill(req, image_id)
raise HTTPRequestEntityTooLarge(explanation=msg, request=req,
content_type='text/plain')
except HTTPError as e:
self._safe_kill(req, image_id)
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
raise e
except Exception as e:
LOG.exception(_("Failed to upload image"))
self._safe_kill(req, image_id)
raise HTTPInternalServerError(request=req)
except exception.Forbidden, e:
msg = _("Forbidden upload attempt: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull, e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageWriteDenied, e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded, e:
msg = _("Denying attempt to upload image larger than %d.")
self._safe_kill(req, image_id)
raise HTTPBadRequest(explanation=msg % CONF.image_size_cap,
def __call__(self):
"""
:return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure
"""
if self.headers.has_key('content-length'):
if int(self.headers['content-length']) >= MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=self.req)
parsed = urlparse(self.url)
if self.proxy:
proxy_parsed = urlparse(self.proxy)
if self._proxy_request_check(parsed.path):
host, port = self.split_netloc(proxy_parsed)
path = self.url
ssl = True if proxy_parsed.scheme == 'https' else False
else:
host, port = self.split_netloc(parsed)
path = parsed.path
ssl = True if parsed.scheme == 'https' else False
self.headers['host'] = '%s:%s' % (host, port)
if self.method == 'PUT' and len(parsed.path.split('/')) >= 5:
if self.headers.has_key('content-length') and int(
self.headers['content-length']) != 0:
if not self.headers.has_key('expect'):
self.headers['expect'] = '100-continue'
chunked = self.req.headers.get('transfer-encoding')
if isinstance(self.req.environ['wsgi.input'], str):
reader = self.req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.chunk_size), '')
else:
data_source = self.req.environ['wsgi.input']
bytes_transferred = 0
try:
conn = self._connect_put_node(host,
port,
self.method,
path,
headers=self.headers,
query_string=parsed.query,
ssl=ssl)
if not conn:
return HTTPServiceUnavailable(request=self.req)
with ContextPool(1) as pool:
conn.failed = False
conn.queue = Queue(10)
pool.spawn(self._send_file, conn, path)
while True:
with ChunkReadTimeout(self.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
conn.queue.put('0\r\n\r\n')
break
except TypeError, err:
self.logger.info('Chunk Read Error: %s' % err)
break
except Exception, err:
self.logger.info('Chunk Read Error: %s' % err)
return HTTPServerError(request=self.req)
bytes_transferred += len(chunk)
if bytes_transferred > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=self.req)
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' %
(len(chunk),
chunk) if chunked else chunk)
def __call__(self):
"""
:return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure
"""
if self.headers.has_key('content-length'):
if int(self.headers['content-length']) >= MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=self.req)
parsed = urlparse(self.url)
if self.proxy:
proxy_parsed = urlparse(self.proxy)
if self._proxy_request_check(parsed.path):
host, port = self.split_netloc(proxy_parsed)
path = self.url
else:
host, port = self.split_netloc(parsed)
path = parsed.path
self.headers['host'] = '%s:%s' % (host, port)
if self.method == 'PUT' and len(parsed.path.split('/')) == 5:
chunked = self.req.headers.get('transfer-encoding')
reader = self.req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.chunk_size), '')
bytes_transferred = 0
# pile = GreenPile()
# pile.spawn(self._connect_server, host, port, self.method, path, self.headers, parsed.query)
# conns = [conn for conn in pile if conn]
# conn = conns[0]
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect_raw(host,
port,
self.method,
path,
headers=self.headers,
query_string=parsed.query)
with ContextPool(1) as pool:
conn.failed = False
conn.queue = Queue(10)
pool.spawn(self._send_file, conn, path)
while True:
with ChunkReadTimeout(self.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
conn.queue.put('0\r\n\r\n')
break
except TypeError, err:
self.logger.info('Chunk Read Error: %s' % err)
break
except Exception, err:
self.logger.info('Chunk Read Error: %s' % err)
return HTTPServerError(request=self.req)
bytes_transferred += len(chunk)
if bytes_transferred > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=self.req)
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' %
(len(chunk),
chunk) if chunked else chunk)
def PUT(self, req):
"""HTTP PUT request handler."""
(container_partition, containers, _junk, req.acl,
req.environ['swift_sync_key'], object_versions) = \
self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
if 'x-delete-after' in req.headers:
try:
x_delete_after = int(req.headers['x-delete-after'])
except ValueError:
return HTTPBadRequest(request=req,
content_type='text/plain',
body='Non-integer X-Delete-After')
req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after)
if 'x-delete-at' in req.headers:
try:
x_delete_at = int(req.headers['x-delete-at'])
if x_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past',
request=req,
content_type='text/plain')
except ValueError:
return HTTPBadRequest(request=req,
content_type='text/plain',
body='Non-integer X-Delete-At')
delete_at_container = str(
x_delete_at / self.app.expiring_objects_container_divisor *
self.app.expiring_objects_container_divisor)
delete_at_part, delete_at_nodes = \
self.app.container_ring.get_nodes(
self.app.expiring_objects_account, delete_at_container)
else:
delete_at_part = delete_at_nodes = None
partition, nodes = self.app.object_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
# do a HEAD request for container sync and checking object versions
if 'x-timestamp' in req.headers or (
object_versions
and not req.environ.get('swift_versioned_copy')):
hreq = Request.blank(req.path_info,
headers={'X-Newest': 'True'},
environ={'REQUEST_METHOD': 'HEAD'})
hresp = self.GETorHEAD_base(hreq, _('Object'), partition, nodes,
hreq.path_info, len(nodes))
# Used by container sync feature
if 'x-timestamp' in req.headers:
try:
req.headers['X-Timestamp'] = \
normalize_timestamp(float(req.headers['x-timestamp']))
if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
float(hresp.environ['swift_x_timestamp']) >= \
float(req.headers['x-timestamp']):
return HTTPAccepted(request=req)
except ValueError:
return HTTPBadRequest(
request=req,
content_type='text/plain',
body='X-Timestamp should be a UNIX timestamp float value; '
'was %r' % req.headers['x-timestamp'])
else:
req.headers['X-Timestamp'] = normalize_timestamp(time.time())
# Sometimes the 'content-type' header exists, but is set to None.
content_type_manually_set = True
if not req.headers.get('content-type'):
guessed_type, _junk = mimetypes.guess_type(req.path_info)
req.headers['Content-Type'] = guessed_type or \
'application/octet-stream'
content_type_manually_set = False
error_response = check_object_creation(req, self.object_name)
if error_response:
return error_response
if object_versions and not req.environ.get('swift_versioned_copy'):
is_manifest = 'x-object-manifest' in req.headers or \
'x-object-manifest' in hresp.headers
if hresp.status_int != HTTP_NOT_FOUND and not is_manifest:
# This is a version manifest and needs to be handled
# differently. First copy the existing data to a new object,
# then write the data from this request to the version manifest
# object.
lcontainer = object_versions.split('/')[0]
prefix_len = '%03x' % len(self.object_name)
lprefix = prefix_len + self.object_name + '/'
ts_source = hresp.environ.get('swift_x_timestamp')
if ts_source is None:
ts_source = time.mktime(
time.strptime(hresp.headers['last-modified'],
'%a, %d %b %Y %H:%M:%S GMT'))
new_ts = normalize_timestamp(ts_source)
vers_obj_name = lprefix + new_ts
copy_headers = {
'Destination': '%s/%s' % (lcontainer, vers_obj_name)
}
copy_environ = {
'REQUEST_METHOD': 'COPY',
'swift_versioned_copy': True
}
copy_req = Request.blank(req.path_info,
headers=copy_headers,
environ=copy_environ)
copy_resp = self.COPY(copy_req)
if is_client_error(copy_resp.status_int):
# missing container or bad permissions
return HTTPPreconditionFailed(request=req)
elif not is_success(copy_resp.status_int):
# could not copy the data, bail
return HTTPServiceUnavailable(request=req)
reader = req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.app.client_chunk_size), '')
source_header = req.headers.get('X-Copy-From')
source_resp = None
if source_header:
source_header = unquote(source_header)
acct = req.path_info.split('/', 2)[1]
if isinstance(acct, unicode):
acct = acct.encode('utf-8')
if not source_header.startswith('/'):
source_header = '/' + source_header
source_header = '/' + acct + source_header
try:
src_container_name, src_obj_name = \
source_header.split('/', 3)[2:]
except ValueError:
return HTTPPreconditionFailed(
request=req,
body='X-Copy-From header must be of the form'
'<container name>/<object name>')
source_req = req.copy_get()
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
self.object_name = src_obj_name
self.container_name = src_container_name
source_resp = self.GET(source_req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp
self.object_name = orig_obj_name
self.container_name = orig_container_name
new_req = Request.blank(req.path_info,
environ=req.environ,
headers=req.headers)
data_source = source_resp.app_iter
new_req.content_length = source_resp.content_length
if new_req.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
return HTTPRequestEntityTooLarge(request=req)
new_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del new_req.headers['X-Copy-From']
if not content_type_manually_set:
new_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if new_req.headers.get('x-fresh-metadata', 'false').lower() \
not in TRUE_VALUES:
for k, v in source_resp.headers.items():
if k.lower().startswith('x-object-meta-'):
new_req.headers[k] = v
for k, v in req.headers.items():
if k.lower().startswith('x-object-meta-'):
new_req.headers[k] = v
req = new_req
node_iter = self.iter_nodes(partition, nodes, self.app.object_ring)
pile = GreenPile(len(nodes))
for container in containers:
nheaders = dict(req.headers.iteritems())
nheaders['Connection'] = 'close'
nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container
nheaders['X-Container-Partition'] = container_partition
nheaders['X-Container-Device'] = container['device']
nheaders['Expect'] = '100-continue'
if delete_at_nodes:
node = delete_at_nodes.pop(0)
nheaders['X-Delete-At-Host'] = '%(ip)s:%(port)s' % node
nheaders['X-Delete-At-Partition'] = delete_at_part
nheaders['X-Delete-At-Device'] = node['device']
pile.spawn(self._connect_put_node, node_iter, partition,
req.path_info, nheaders, self.app.logger.thread_locals)
conns = [conn for conn in pile if conn]
if len(conns) <= len(nodes) / 2:
self.app.logger.error(
_('Object PUT returning 503, %(conns)s/%(nodes)s '
'required connections'), {
'conns': len(conns),
'nodes': len(nodes) // 2 + 1
})
return HTTPServiceUnavailable(request=req)
chunked = req.headers.get('transfer-encoding')
bytes_transferred = 0
try:
with ContextPool(len(nodes)) as pool:
for conn in conns:
conn.failed = False
conn.queue = Queue(self.app.put_queue_depth)
pool.spawn(self._send_file, conn, req.path)
while True:
with ChunkReadTimeout(self.app.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
[conn.queue.put('0\r\n\r\n') for conn in conns]
break
bytes_transferred += len(chunk)
if bytes_transferred > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
for conn in list(conns):
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' %
(len(chunk),
chunk) if chunked else chunk)
else:
conns.remove(conn)
if len(conns) <= len(nodes) / 2:
self.app.logger.error(
_('Object PUT exceptions during'
' send, %(conns)s/%(nodes)s required connections'
), {
'conns': len(conns),
'nodes': len(nodes) / 2 + 1
})
return HTTPServiceUnavailable(request=req)
for conn in conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
conns = [conn for conn in conns if not conn.failed]
except ChunkReadTimeout, err:
self.app.logger.warn(_('ERROR Client read timeout (%ss)'),
err.seconds)
self.app.logger.increment('client_timeouts')
return HTTPRequestTimeout(request=req)