def __call__(self, env, start_response):
if env['REQUEST_METHOD'] == 'GET':
if self.status == 200:
start_response(Response().status,
[('Content-Type', 'text/xml')])
json_pattern = [
'"name":%s', '"last_modified":%s', '"hash":%s',
'"bytes":%s'
]
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
for b in self.objects:
name = simplejson.dumps(b[0])
time = simplejson.dumps(b[1])
json_out.append(json_pattern % (name, time, b[2], b[3]))
account_list = '[' + ','.join(json_out) + ']'
return account_list
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 403:
start_response(HTTPForbidden().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(HTTPCreated().status, [])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 403:
start_response(HTTPForbidden().status, [])
elif self.status == 202:
start_response(HTTPAccepted().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'POST':
if self.status == 204:
start_response(HTTPNoContent().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent().status, [])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 403:
start_response(HTTPForbidden().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
elif self.status == 409:
start_response(HTTPConflict().status, [])
else:
start_response(HTTPBadRequest().status, [])
return []
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
"""
Handles requests to /info
Should return a WSGI-style callable (such as swob.Response).
:param req: swob.Request object
"""
if not self.expose_info:
return HTTPForbidden(request=req)
admin_request = False
sig = req.params.get('swiftinfo_sig', '')
expires = req.params.get('swiftinfo_expires', '')
if sig != '' or expires != '':
admin_request = True
if not self.admin_key:
return HTTPForbidden(request=req)
try:
expires = int(expires)
except ValueError:
return HTTPUnauthorized(request=req)
if expires < time():
return HTTPUnauthorized(request=req)
valid_sigs = []
for method in self.allowed_hmac_methods[req.method]:
valid_sigs.append(
get_hmac(method, '/info', expires, self.admin_key))
# While it's true that any() will short-circuit, this doesn't
# affect the timing-attack resistance since the only way this will
# short-circuit is when a valid signature is passed in.
is_valid_hmac = any(
streq_const_time(valid_sig, sig) for valid_sig in valid_sigs)
if not is_valid_hmac:
return HTTPUnauthorized(request=req)
headers = {}
if 'Origin' in req.headers:
headers['Access-Control-Allow-Origin'] = req.headers['Origin']
headers['Access-Control-Expose-Headers'] = ', '.join(
['x-trans-id'])
info = json.dumps(
get_swift_info(admin=admin_request,
disallowed_sections=self.disallowed_sections))
return HTTPOk(request=req,
headers=headers,
body=info,
content_type='application/json; charset=UTF-8')
def __call__(self, env, start_response):
req = Request(env)
if env['REQUEST_METHOD'] == 'GET' or env['REQUEST_METHOD'] == 'HEAD':
if self.status == 200:
if 'HTTP_RANGE' in env:
resp = Response(request=req,
body=self.object_body,
conditional_response=True)
return resp(env, start_response)
start_response(
Response(request=req).status,
self.response_headers.items())
if env['REQUEST_METHOD'] == 'GET':
return self.object_body
elif self.status == 401:
start_response(HTTPUnauthorized(request=req).status, [])
elif self.status == 403:
start_response(HTTPForbidden(request=req).status, [])
elif self.status == 404:
start_response(HTTPNotFound(request=req).status, [])
else:
start_response(HTTPBadRequest(request=req).status, [])
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(
HTTPCreated(request=req).status,
[('etag', self.response_headers['etag'])])
elif self.status == 401:
start_response(HTTPUnauthorized(request=req).status, [])
elif self.status == 403:
start_response(HTTPForbidden(request=req).status, [])
elif self.status == 404:
start_response(HTTPNotFound(request=req).status, [])
elif self.status == 413:
start_response(
HTTPRequestEntityTooLarge(request=req).status, [])
else:
start_response(HTTPBadRequest(request=req).status, [])
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent(request=req).status, [])
elif self.status == 401:
start_response(HTTPUnauthorized(request=req).status, [])
elif self.status == 403:
start_response(HTTPForbidden(request=req).status, [])
elif self.status == 404:
start_response(HTTPNotFound(request=req).status, [])
else:
start_response(HTTPBadRequest(request=req).status, [])
return []
def google_checkout(self, env, query_dict):
type = query_dict.get('type', None)
if not type is None:
type = type[0]
try:
self.item_names[type]
except IndexError:
return HTTPForbidden()
account = query_dict['account'][0]
state = query_dict['state'][0]
req = Request.blank('/v1/%s' % account)
req.method = 'HEAD'
resp = req.get_response(self.app)
if resp.status_int >= 300:
return HTTPNotFound()
services = {}
if self.service_metadata_key in resp.headers:
try:
services = json.loads(
resp.headers[self.service_metadata_key])
existing_service = services.get(self.item_names[type],
None)
if existing_service:
return Response(
status=302,
headers={
'location':
'%s%s?account=%s&msg=service_exists' %
(self.service_endpoint, state, account)
})
except Exception:
self.logger.exception(
'Exception when parsing x-account-meta-services for %s'
% account)
return HTTPForbidden()
services[self.item_names[type]] = 'pending'
req = Request.blank('/v1/%s' % account)
req.method = 'POST'
req.headers[self.service_metadata_key] = json.dumps(services)
req.get_response(self.app)
loc = self.create_checkout_uri(self.item_names[type],
self.item_descriptions[type],
self.item_prices[type],
self.item_quantity,
self.item_currency)
if loc:
return Response(status=302, headers={'location': loc})
req = Request(env)
return Response(request=req)
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
"""
Handles requests to /info
Should return a WSGI-style callable (such as swob.Response).
:param req: swob.Request object
"""
if not self.expose_info:
return HTTPForbidden(request=req)
admin_request = False
sig = req.params.get('swiftinfo_sig', '')
expires = req.params.get('swiftinfo_expires', '')
if sig != '' or expires != '':
admin_request = True
if not self.admin_key:
return HTTPForbidden(request=req)
try:
expires = int(expires)
except ValueError:
return HTTPUnauthorized(request=req)
if expires < time():
return HTTPUnauthorized(request=req)
valid_sigs = []
for method in self.allowed_hmac_methods[req.method]:
valid_sigs.append(
get_hmac(method, '/info', expires, self.admin_key))
if sig not in valid_sigs:
return HTTPUnauthorized(request=req)
headers = {}
if 'Origin' in req.headers:
headers['Access-Control-Allow-Origin'] = req.headers['Origin']
headers['Access-Control-Expose-Headers'] = ', '.join(
['x-trans-id'])
info = json.dumps(
get_swift_info(admin=admin_request,
disallowed_sections=self.disallowed_sections))
return HTTPOk(request=req,
headers=headers,
body=info,
content_type='application/json; charset=UTF-8')
def __call__(self, env, start_response):
qs = env.get('QUERY_STRING', None)
if env.get('PATH_INFO', '').startswith(self.billing_prefix):
if qs:
account = parse_qs(qs).get('account', None)
if account:
return self.google_checkout(env,
parse_qs(qs))(env,
start_response)
elif env.get('PATH_INFO', '').startswith(self.notification_prefix):
if qs:
type = parse_qs(qs).get('type', None)
if type:
type = type[0]
if 'new-order-notification' in type:
return self.new_order(env)(env, start_response)
elif 'risk-information-notification' in type:
return self.risk_info(env)(env, start_response)
elif 'order-state-change-notification' in type:
return self.order_changed(env)(env, start_response)
elif 'charge-amount-notification' in type:
return self.charge_amount(env)(env, start_response)
else:
return self.unhandled(env)(env, start_response)
return HTTPForbidden()
def denied_response(self, req):
if req.remote_user:
self.logger.increment('forbidden')
return HTTPForbidden(request=req)
else:
self.logger.increment('unauthorized')
return HTTPUnauthorized(request=req)
def PUT(self, req):
"""HTTP PUT request handler."""
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Container name length of %d longer than %d' % \
(len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name,
autocreate=self.app.account_autocreate)
if self.app.max_containers_per_account > 0 and \
container_count >= self.app.max_containers_per_account and \
self.account_name not in self.app.max_containers_whitelist:
resp = HTTPForbidden(request=req)
resp.body = 'Reached container limit of %s' % \
self.app.max_containers_per_account
return resp
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts)
if self.app.memcache:
cache_key = get_container_memcache_key(self.account_name,
self.container_name)
self.app.memcache.delete(cache_key)
resp = self.make_requests(req, self.app.container_ring,
container_partition, 'PUT', req.path_info,
headers)
return resp
def decrypt_resp_headers(self, put_keys, post_keys):
"""
Find encrypted headers and replace with the decrypted versions.
:param put_keys: a dict of decryption keys used for object PUT.
:param post_keys: a dict of decryption keys used for object POST.
:return: A list of headers with any encrypted headers replaced by their
decrypted values.
:raises HTTPInternalServerError: if any error occurs while decrypting
headers
:raises HTTPForbidden: if the decryption key is invalid
"""
mod_hdr_pairs = []
if put_keys:
# Decrypt plaintext etag and place in Etag header for client
# response
etag_header = 'X-Object-Sysmeta-Crypto-Etag'
encrypted_etag = self._response_header_value(etag_header)
decrypted_etag = None
if encrypted_etag and 'object' in put_keys:
decrypted_etag = self._decrypt_header(etag_header,
encrypted_etag,
put_keys['object'],
required=True)
mod_hdr_pairs.append(('Etag', decrypted_etag))
etag_header = get_container_update_override_key('etag')
encrypted_etag = self._response_header_value(etag_header)
if encrypted_etag and 'container' in put_keys:
dcrypt_etag_override = self._decrypt_header(
etag_header, encrypted_etag, put_keys['container'])
if decrypted_etag and dcrypt_etag_override != decrypted_etag:
self.app.logger.debug('Failed ETag verification')
raise HTTPForbidden('Invalid key')
mod_hdr_pairs.append((etag_header, dcrypt_etag_override))
# The real swift saves the cyphered ETag in the 'ETag' field,
# whereas we store the ETag of the cyphered object.
# The ETag of the cyphered object is of no use for previous
# middlewares, so we replace it with the plaintext ETag.
mod_hdr_pairs.append(('ETag', dcrypt_etag_override))
# Decrypt all user metadata. Encrypted user metadata values are stored
# in the x-object-transient-sysmeta-crypto-meta- namespace. Those are
# decrypted and moved back to the x-object-meta- namespace. Prior to
# decryption, the response should have no x-object-meta- headers, but
# if it does then they will be overwritten by any decrypted headers
# that map to the same x-object-meta- header names i.e. decrypted
# headers win over unexpected, unencrypted headers.
try:
if post_keys:
mod_hdr_pairs.extend(self.decrypt_user_metadata(post_keys))
mod_hdr_names = {h.lower() for h, v in mod_hdr_pairs}
mod_hdr_pairs.extend([(h, v) for h, v in self._response_headers
if h.lower() not in mod_hdr_names])
except KeyError:
self.app.logger.debug('Not able to decrypt user metadata')
return mod_hdr_pairs
def __call__(self, request):
if request.method not in ("POST", "PUT"):
return self.app
try:
request.split_path(2, 4, rest_with_last=True)
except ValueError:
return self.app
new_quota = request.headers.get('X-Account-Meta-Quota-Bytes')
remove_quota = request.headers.get('X-Remove-Account-Meta-Quota-Bytes')
if remove_quota:
new_quota = 0 # X-Remove dominates if both are present
if request.environ.get('reseller_request') is True:
if new_quota and not new_quota.isdigit():
return HTTPBadRequest()
return self.app
# deny quota set for non-reseller
if new_quota is not None:
return HTTPForbidden()
account_info = get_account_info(request.environ, self.app)
if not account_info or not account_info['bytes']:
return self.app
new_size = int(account_info['bytes']) + (request.content_length or 0)
quota = int(account_info['meta'].get('quota-bytes', -1))
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge()
return self.app
def handle_login(self, req, code, state):
self.storage_driver = LiteAuthStorage(req.environ, self.prefix)
oauth_client = self.provider.create_for_token(self.conf, code)
token = oauth_client.access_token
if not token:
req.response = HTTPUnauthorized(request=req)
return req.response
user_info = oauth_client.userinfo
if not user_info:
req.response = HTTPForbidden(request=req)
return req.response
account_id = '%s:%s' % (self.prefix + user_info.get('id'),
user_info.get('email'))
self.storage_driver.store_id(account_id, token,
oauth_client.expires_in)
return Response(request=req,
status=302,
headers={
'x-auth-token':
token,
'x-storage-token':
token,
'x-auth-token-expires':
oauth_client.expires_in,
'x-storage-url':
self.auth_endpoint,
'location':
'%s%s?account=%s' %
(self.service_domain, state or '/', account_id)
})
def __call__(self, request):
if request.method not in ("POST", "PUT"):
return self.app
try:
ver, account, container, obj = request.split_path(
2, 4, rest_with_last=True)
except ValueError:
return self.app
if not container:
# account request, so we pay attention to the quotas
new_quota = request.headers.get('X-Account-Meta-Quota-Bytes')
remove_quota = request.headers.get(
'X-Remove-Account-Meta-Quota-Bytes')
else:
# container or object request; even if the quota headers are set
# in the request, they're meaningless
new_quota = remove_quota = None
if remove_quota:
new_quota = 0 # X-Remove dominates if both are present
if request.environ.get('reseller_request') is True:
if new_quota and not new_quota.isdigit():
return HTTPBadRequest()
return self.app
# deny quota set for non-reseller
if new_quota is not None:
return HTTPForbidden()
if obj and request.method == "POST" or not obj:
return self.app
copy_from = request.headers.get('X-Copy-From')
content_length = (request.content_length or 0)
if obj and copy_from:
path = '/' + ver + '/' + account + '/' + copy_from.lstrip('/')
object_info = get_object_info(request.environ, self.app, path)
if not object_info or not object_info['length']:
content_length = 0
else:
content_length = int(object_info['length'])
account_info = get_account_info(request.environ, self.app)
if not account_info or not account_info['bytes']:
return self.app
new_size = int(account_info['bytes']) + content_length
quota = int(account_info['meta'].get('quota-bytes', -1))
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge()
return self.app
def denied_response(self, req):
"""
Returns a standard WSGI response callable with the status of 403 or 401
depending on whether the REMOTE_USER is set or not.
"""
if req.remote_user:
return HTTPForbidden(request=req)
else:
return HTTPUnauthorized(request=req)
def _should_block(self, request):
# Currently, we have only one reason to block
# requests at such an early stage of the processing:
# we block requests with referer that have the internal prefix
# of:
if request.referer and REFERER_PREFIX in request.referer:
msg = 'Referrer containing %s is not allowed' % REFERER_PREFIX
self.logger.debug(msg)
raise HTTPForbidden(msg.encode('utf8'), request=self.request)
def denied_response(self, req):
"""
Returns a standard WSGI response callable with the status of 403 or 401
depending on whether the REMOTE_USER is set or not.
"""
if req.remote_user:
self.logger.increment('forbidden')
return HTTPForbidden(request=req)
else:
return HTTPSeeOther(location=self.ext_authentication_url)
def validate_first_segment(self):
try:
return super(OioSegmentedIterable, self).validate_first_segment()
except exceptions.SegmentError as err:
if 'got 403 while retrieving' in err.args[0]:
raise HTTPForbidden(request=self.req)
elif 'got 400 while retrieving' in err.args[0]:
raise HTTPBadRequest(request=self.req)
else:
raise
def __call__(self, env, start_response):
req = Request(env)
if req.method in self.verb_acl:
remote = get_remote_client(req)
for block in self.verb_acl[req.method]:
if remote.startswith(block):
break
else:
raise HTTPForbidden(request=req,
body='Forbidden method for %s' % remote)
return self.app(env, start_response)
def __call__(self, env, start_response):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
replication_method = getattr(method, 'replication', False)
if (self.replication_server is not None and
self.replication_server != replication_method):
raise AttributeError('Not allowed method.')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)
except DiskFileCollision:
res = HTTPForbidden(request=req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(_(
'ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = time.time() - start_time
if self.log_requests:
log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000',
time.gmtime()),
req.method, req.path, res.status.split()[0],
res.content_length or '-', req.referer or '-',
req.headers.get('x-trans-id', '-'),
req.user_agent or '-',
trans_time)
if req.method in ('REPLICATE', 'REPLICATION') or \
'X-Backend-Replication' in req.headers:
self.logger.debug(log_line)
else:
self.logger.info(log_line)
if req.method in ('PUT', 'DELETE'):
slow = self.slow - trans_time
if slow > 0:
sleep(slow)
return res(env, start_response)
def PUT(self, req):
"""HTTP PUT request handler."""
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
policy_index = self._convert_policy_to_index(req)
if policy_index is None:
# make sure all backend servers get the same default policy
policy_index = POLICIES.default.idx
policy = POLICIES[policy_index]
if policy.is_deprecated:
resp = HTTPBadRequest(request=req)
resp.body = 'Storage Policy %r is deprecated' % \
(policy.name)
return resp
if not req.environ.get('swift_owner'):
for key in self.app.swift_owner_headers:
req.headers.pop(key, None)
if len(self.container_name) > constraints.MAX_CONTAINER_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Container name length of %d longer than %d' % \
(len(self.container_name),
constraints.MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts and self.app.account_autocreate:
self.autocreate_account(req.environ, self.account_name)
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
if self.app.max_containers_per_account > 0 and \
container_count >= self.app.max_containers_per_account and \
self.account_name not in self.app.max_containers_whitelist:
resp = HTTPForbidden(request=req)
resp.body = 'Reached container limit of %s' % \
self.app.max_containers_per_account
return resp
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts,
policy_index)
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
req, self.app.container_ring,
container_partition, 'PUT', req.swift_entity_path, headers)
return resp
def PUT(self, req):
"""HTTP PUT request handler."""
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
policy_index = self._convert_policy_to_index(req)
if not req.environ.get('swift_owner'):
for key in self.app.swift_owner_headers:
req.headers.pop(key, None)
if req.environ.get('reseller_request', False) and \
'X-Container-Sharding' in req.headers:
req.headers[get_sys_meta_prefix('container') + 'Sharding'] = \
str(config_true_value(req.headers['X-Container-Sharding']))
length_limit = self.get_name_length_limit()
if len(self.container_name) > length_limit:
body = 'Container name length of %d longer than %d' % (len(
self.container_name), length_limit)
resp = HTTPBadRequest(request=req, body=body)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts and self.app.account_autocreate:
if not self.autocreate_account(req, self.account_name):
return HTTPServiceUnavailable(request=req)
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
if 0 < self.app.max_containers_per_account <= container_count and \
self.account_name not in self.app.max_containers_whitelist:
container_info = \
self.container_info(self.account_name, self.container_name,
req)
if not is_success(container_info.get('status')):
body = 'Reached container limit of %s' % (
self.app.max_containers_per_account, )
resp = HTTPForbidden(request=req, body=body)
return resp
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts,
policy_index)
resp = self.make_requests(req, self.app.container_ring,
container_partition, 'PUT',
req.swift_entity_path, headers)
clear_info_cache(self.app, req.environ, self.account_name,
self.container_name)
return resp
def create_error_response(error, message):
if error == 400:
response = HTTPBadRequest(body=message)
elif error == 401:
response = HTTPUnauthorized(body=message)
elif error == 403:
response = HTTPForbidden(body=message)
elif error == 404:
response = HTTPNotFound(body=message)
elif error == 405:
response = HTTPMethodNotAllowed(body=message)
else:
response = HTTPServerError(body=message)
return response
def PUT(self, req):
"""HTTP PUT request handler."""
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
policy_index = self._convert_policy_to_index(req)
if not req.environ.get('swift_owner'):
for key in self.app.swift_owner_headers:
req.headers.pop(key, None)
if len(self.container_name) > constraints.MAX_CONTAINER_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Container name length of %d longer than %d' % \
(len(self.container_name),
constraints.MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts and self.app.account_autocreate:
if not self.autocreate_account(req, self.account_name):
return HTTPServerError(request=req)
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
if 0 < self.app.max_containers_per_account <= container_count and \
self.account_name not in self.app.max_containers_whitelist:
container_info = \
self.container_info(self.account_name, self.container_name,
req)
if not is_success(container_info.get('status')):
resp = HTTPForbidden(request=req)
resp.body = 'Reached container limit of %s' % \
self.app.max_containers_per_account
return resp
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts,
policy_index)
resp = self.make_requests(req, self.app.container_ring,
container_partition, 'PUT',
req.swift_entity_path, headers)
clear_info_cache(self.app, req.environ, self.account_name,
self.container_name)
return resp
def __call__(self, env, start_response):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
replication_method = getattr(method, 'replication', False)
if (self.replication_server is not None and
self.replication_server != replication_method):
raise AttributeError('Not allowed method.')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)
except DiskFileCollision:
res = HTTPForbidden(request=req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(_(
'ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = time.time() - start_time
if self.log_requests:
log_line = get_log_line(req, res, trans_time, '')
if req.method in ('REPLICATE', 'REPLICATION') or \
'X-Backend-Replication' in req.headers:
self.logger.debug(log_line)
else:
self.logger.info(log_line)
if req.method in ('PUT', 'DELETE'):
slow = self.slow - trans_time
if slow > 0:
sleep(slow)
my_debug('returning from __call__ or objserver', res)
my_debug('returnign from __call__ of objserver', start_response)
return res(env, start_response)
def PUT(self, req):
"""HTTP PUT request handler."""
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Container name length of %d longer than %d' % \
(len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name,
autocreate=self.app.account_autocreate)
if self.app.max_containers_per_account > 0 and \
container_count >= self.app.max_containers_per_account and \
self.account_name not in self.app.max_containers_whitelist:
resp = HTTPForbidden(request=req)
resp.body = 'Reached container limit of %s' % \
self.app.max_containers_per_account
return resp
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = []
for account in accounts:
nheaders = {
'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'X-Account-Host': '%(ip)s:%(port)s' % account,
'X-Account-Partition': account_partition,
'X-Account-Device': account['device'],
'Connection': 'close'
}
self.transfer_headers(req.headers, nheaders)
headers.append(nheaders)
if self.app.memcache:
cache_key = get_container_memcache_key(self.account_name,
self.container_name)
self.app.memcache.delete(cache_key)
resp = self.make_requests(req, self.app.container_ring,
container_partition, 'PUT', req.path_info,
headers)
return resp
def __call__(self, env, start_response):
if self.status == 200:
start_response(Response().status, [('Content-Type', 'text/xml')])
json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
for b in self.buckets:
name = simplejson.dumps(b[0])
json_out.append(json_pattern % (name, b[1], b[2]))
account_list = '[' + ','.join(json_out) + ']'
return account_list
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 403:
start_response(HTTPForbidden().status, [])
else:
start_response(HTTPBadRequest().status, [])
return []
def PUT(self, req):
"""HTTP PUT request handler."""
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
if not req.environ.get('swift_owner'):
for key in self.app.swift_owner_headers:
req.headers.pop(key, None)
if len(self.container_name) > constraints.MAX_CONTAINER_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Container name length of %d longer than %d' % \
(len(self.container_name),
constraints.MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts and self.app.account_autocreate:
self.autocreate_account(req, self.account_name)
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
if self.app.max_containers_per_account > 0 and \
container_count >= self.app.max_containers_per_account and \
self.account_name not in self.app.max_containers_whitelist:
container_info = \
self.container_info(self.account_name, self.container_name,
req)
if not is_success(container_info.get('status')):
resp = HTTPForbidden(request=req)
resp.body = 'Reached container limit of %s' % \
self.app.max_containers_per_account
return resp
headers = self.generate_request_headers(req, transfer=True)
clear_info_cache(self.app, req.environ, self.account_name,
self.container_name)
resp = self.get_container_create_resp(req, headers)
return resp
def handle_request(self, env, req):
"""
Handles the POST /origin/.prep call for preparing the backing store
Swift cluster for use with the origin subsystem. Can only be called by
.origin_admin
:param req: The swob.Request to process.
:returns: swob.Response, 204 on success
"""
if not self.is_origin_admin(req):
return HTTPForbidden(request=req)
try:
vsn, account = split_path(req.path, 2, 2)
except ValueError:
return HTTPBadRequest(request=req)
if account == '.prep':
path = '/v1/%s' % self.origin_account
resp = make_pre_authed_request(req.environ,
'PUT',
path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if resp.status_int // 100 != 2:
raise Exception(
'Could not create the main origin account: %s %s' %
(path, resp.status))
for i in xrange(self.num_hash_cont):
cont_name = '.hash_%d' % i
path = '/v1/%s/%s' % (self.origin_account, cont_name)
resp = make_pre_authed_request(
req.environ,
'PUT',
path,
agent='SwiftOrigin',
swift_source='SOS').get_response(self.app)
if resp.status_int // 100 != 2:
raise Exception('Could not create %s container: %s %s' %
(cont_name, path, resp.status))
return HTTPNoContent(request=req)
return HTTPNotFound(request=req)
req.headers['x-object-count'],
req.headers['x-bytes-used'])
self.logger.timing_since('PUT.timing', start_time)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = normalize_timestamp(req.headers['x-timestamp'])
if not os.path.exists(broker.db_file):
broker.initialize(timestamp)
created = True
elif broker.is_status_deleted():
self.logger.timing_since('PUT.timing', start_time)
return HTTPForbidden(request=req, body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp)
if broker.is_deleted():
self.logger.increment('PUT.errors')
return HTTPConflict(request=req)
metadata = {}
metadata.update((key, (value, timestamp))
for key, value in req.headers.iteritems()
if key.lower().startswith('x-account-meta-'))
if metadata:
broker.update_metadata(metadata)
self.logger.timing_since('PUT.timing', start_time)
if created:
return HTTPCreated(request=req)
def handle_request(self, req):
"""
Entry point for proxy server.
Should return a WSGI-style callable (such as swob.Response).
:param req: swob.Request object
"""
try:
self.logger.set_statsd_prefix('proxy-server')
if req.content_length and req.content_length < 0:
self.logger.increment('errors')
return HTTPBadRequest(request=req,
body='Invalid Content-Length')
try:
if not check_utf8(req.path_info):
self.logger.increment('errors')
return HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
except UnicodeError:
self.logger.increment('errors')
return HTTPPreconditionFailed(
request=req, body='Invalid UTF8 or contains NULL')
try:
controller, path_parts = self.get_controller(req)
p = req.path_info
if isinstance(p, six.text_type):
p = p.encode('utf-8')
except APIVersionError:
self.logger.increment('errors')
return HTTPBadRequest(request=req)
except ValueError:
self.logger.increment('errors')
return HTTPNotFound(request=req)
if not controller:
self.logger.increment('errors')
return HTTPPreconditionFailed(request=req, body='Bad URL')
if self.deny_host_headers and \
req.host.split(':')[0] in self.deny_host_headers:
return HTTPForbidden(request=req, body='Invalid host header')
self.logger.set_statsd_prefix('proxy-server.' +
controller.server_type.lower())
controller = controller(self, **path_parts)
if 'swift.trans_id' not in req.environ:
# if this wasn't set by an earlier middleware, set it now
trans_id_suffix = self.trans_id_suffix
trans_id_extra = req.headers.get('x-trans-id-extra')
if trans_id_extra:
trans_id_suffix += '-' + trans_id_extra[:32]
trans_id = generate_trans_id(trans_id_suffix)
req.environ['swift.trans_id'] = trans_id
self.logger.txn_id = trans_id
req.headers['x-trans-id'] = req.environ['swift.trans_id']
controller.trans_id = req.environ['swift.trans_id']
self.logger.client_ip = get_remote_client(req)
try:
handler = getattr(controller, req.method)
getattr(handler, 'publicly_accessible')
except AttributeError:
allowed_methods = getattr(controller, 'allowed_methods', set())
return HTTPMethodNotAllowed(
request=req, headers={'Allow': ', '.join(allowed_methods)})
old_authorize = None
if 'swift.authorize' in req.environ:
# We call authorize before the handler, always. If authorized,
# we remove the swift.authorize hook so isn't ever called
# again. If not authorized, we return the denial unless the
# controller's method indicates it'd like to gather more
# information and try again later.
resp = req.environ['swift.authorize'](req)
if not resp and not req.headers.get('X-Copy-From-Account') \
and not req.headers.get('Destination-Account'):
# No resp means authorized, no delayed recheck required.
old_authorize = req.environ['swift.authorize']
else:
# Response indicates denial, but we might delay the denial
# and recheck later. If not delayed, return the error now.
if not getattr(handler, 'delay_denial', None):
return resp
# Save off original request method (GET, POST, etc.) in case it
# gets mutated during handling. This way logging can display the
# method the client actually sent.
req.environ['swift.orig_req_method'] = req.method
try:
if old_authorize:
req.environ.pop('swift.authorize', None)
return handler(req)
finally:
if old_authorize:
req.environ['swift.authorize'] = old_authorize
except HTTPException as error_response:
return error_response
except (Exception, Timeout):
self.logger.exception(_('ERROR Unhandled exception in request'))
return HTTPServerError(request=req)
def __call__(self, env, start_response):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which have not been marked 'public'
try:
if req.method not in self.allowed_methods:
raise AttributeError('Not allowed method.')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
method = getattr(self, req.method)
res = method(req)
except DiskFileCollision:
res = HTTPForbidden(request=req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(
_('ERROR __call__ error with %(method)s'
' %(path)s '), {
'method': req.method,
'path': req.path
})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = time.time() - start_time
if self.log_requests:
log_line = get_log_line(req, res, trans_time, '')
if req.method in ('REPLICATE', 'REPLICATION') or \
'X-Backend-Replication' in req.headers:
self.logger.debug(log_line)
else:
self.logger.info(log_line)
if req.method in ('PUT', 'DELETE'):
slow = self.slow - trans_time
if slow > 0:
sleep(slow)
# To be able to zero-copy send the object, we need a few things.
# First, we have to be responding successfully to a GET, or else we're
# not sending the object. Second, we have to be able to extract the
# socket file descriptor from the WSGI input object. Third, the
# diskfile has to support zero-copy send.
#
# There's a good chance that this could work for 206 responses too,
# but the common case is sending the whole object, so we'll start
# there.
if req.method == 'GET' and res.status_int == 200 and \
isinstance(env['wsgi.input'], wsgi.Input):
app_iter = getattr(res, 'app_iter', None)
checker = getattr(app_iter, 'can_zero_copy_send', None)
if checker and checker():
# For any kind of zero-copy thing like sendfile or splice, we
# need the file descriptor. Eventlet doesn't provide a clean
# way of getting that, so we resort to this.
wsock = env['wsgi.input'].get_socket()
wsockfd = wsock.fileno()
# Don't call zero_copy_send() until after we force the HTTP
# headers out of Eventlet and into the socket.
def zero_copy_iter():
# If possible, set TCP_CORK so that headers don't
# immediately go on the wire, but instead, wait for some
# response body to make the TCP frames as large as
# possible (and hence as few packets as possible).
#
# On non-Linux systems, we might consider TCP_NODELAY, but
# since the only known zero-copy-capable diskfile uses
# Linux-specific syscalls, we'll defer that work until
# someone needs it.
if hasattr(socket, 'TCP_CORK'):
wsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK,
1)
yield EventletPlungerString()
try:
app_iter.zero_copy_send(wsockfd)
except Exception:
self.logger.exception("zero_copy_send() blew up")
raise
yield ''
# Get headers ready to go out
res(env, start_response)
return zero_copy_iter()
else:
return res(env, start_response)
else:
return res(env, start_response)
