def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)
except (Exception, Timeout):
self.logger.exception(_('ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = '%.4f' % (time.time() - start_time)
log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000',
time.gmtime()),
req.method, req.path,
res.status.split()[0], res.content_length or '-',
req.headers.get('x-trans-id', '-'),
req.referer or '-', req.user_agent or '-',
trans_time)
if req.method.upper() == 'REPLICATE':
self.logger.debug(log_message)
else:
self.logger.info(log_message)
return res(env, start_response)
def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which have not been marked 'public'
if req.method not in self.allowed_methods:
res = HTTPMethodNotAllowed()
else:
res = getattr(self, req.method)(req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(
_('ERROR __call__ error with %(method)s %(path)s '), {
'method': req.method,
'path': req.path
})
res = HTTPInternalServerError(body=traceback.format_exc())
if self.log_requests:
trans_time = time.time() - start_time
log_message = get_log_line(req, res, trans_time, '')
if req.method.upper() == 'REPLICATE':
self.logger.debug(log_message)
else:
self.logger.info(log_message)
return res(env, start_response)
def get_unwrapped_key(self, crypto_meta, wrapping_key):
"""
Get a wrapped key from crypto-meta and unwrap it using the provided
wrapping key.
:param crypto_meta: a dict of crypto-meta
:param wrapping_key: key to be used to decrypt the wrapped key
:return: an unwrapped key
:raises EncryptionException: if the crypto-meta has no wrapped key or
the unwrapped key is invalid
"""
try:
return self.crypto.unwrap_key(wrapping_key,
crypto_meta['body_key'])
except KeyError as err:
self.logger.error(
_('Error decrypting %(resp_type)s: Missing %(key)s'), {
'resp_type': self.server_type,
'key': err
})
except ValueError as err:
self.logger.error(_('Error decrypting %(resp_type)s: %(reason)s'),
{
'resp_type': self.server_type,
'reason': err
})
raise HTTPInternalServerError(body='Error decrypting %s' %
self.server_type,
content_type='text/plain')
def handle_get(self, req, start_response):
app_resp = self._app_call(req.environ)
if is_success(self._get_status_int()):
# only decrypt body of 2xx responses
handler = keys = None
for header, value in self._response_headers:
if header.lower() == 'content-type' and \
value.split(';', 1)[0] == 'application/json':
handler = self.process_json_resp
keys = self.get_decryption_keys(req)
if handler and keys:
try:
app_resp = handler(keys['container'], app_resp)
except EncryptionException as err:
self.logger.error(
_("Error decrypting container listing: %s"), err)
raise HTTPInternalServerError(
body='Error decrypting container listing',
content_type='text/plain')
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return app_resp
def _decrypt_header(self, header, value, key, required=False):
"""
Attempt to decrypt a header value that may be encrypted.
:param header: the header name
:param value: the header value
:param key: decryption key
:param required: if True then the header is required to be decrypted
and an HTTPInternalServerError will be raised if the
header cannot be decrypted due to missing crypto meta.
:return: decrypted value or the original value if it was not encrypted.
:raises HTTPInternalServerError: if an error occurred during decryption
or if the header value was required to
be decrypted but crypto meta was not
found.
"""
try:
return self.decrypt_value_with_meta(value, key, required)
except EncryptionException as err:
self.logger.error(
_("Error decrypting header %(header)s: %(error)s"), {
'header': header,
'error': err
})
raise HTTPInternalServerError(body='Error decrypting header',
content_type='text/plain')
def handle(self, req, start_response):
app_resp = self._app_call(req.environ)
try:
put_crypto_meta = self._read_crypto_meta(
'X-Object-Sysmeta-Crypto-Body-Meta', True)
put_keys = self.get_decryption_keys(req, put_crypto_meta)
post_crypto_meta = self._read_crypto_meta(
'X-Object-Transient-Sysmeta-Crypto-Meta', False)
post_keys = self.get_decryption_keys(req, post_crypto_meta)
except EncryptionException as err:
self.logger.error(
"Error decrypting object: %s",
err)
raise HTTPInternalServerError(
body='Error decrypting object',
content_type='text/plain')
if put_keys is None and post_keys is None:
# skip decryption
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return app_resp
mod_resp_headers = self.decrypt_resp_headers(put_keys, post_keys)
if put_crypto_meta and req.method == 'GET' and \
is_success(self._get_status_int()):
# 2xx response and encrypted body
body_key = self.get_unwrapped_key(
put_crypto_meta, put_keys['object'])
content_type, content_type_attrs = parse_content_type(
self._response_header_value('Content-Type'))
if (self._get_status_int() == 206 and
content_type == 'multipart/byteranges'):
boundary = wsgi_to_bytes(dict(content_type_attrs)["boundary"])
resp_iter = self.multipart_response_iter(
app_resp, boundary, body_key, put_crypto_meta)
else:
offset = 0
content_range = self._response_header_value('Content-Range')
if content_range:
# Determine offset within the whole object if ranged GET
offset, end, total = parse_content_range(content_range)
resp_iter = self.response_iter(
app_resp, body_key, put_crypto_meta, offset)
else:
# don't decrypt body of unencrypted or non-2xx responses
resp_iter = app_resp
mod_resp_headers = purge_crypto_sysmeta_headers(mod_resp_headers)
start_response(self._response_status, mod_resp_headers,
self._response_exc_info)
return resp_iter
def _create_entry(self, keys, path, headers):
try:
trans = self.conn.pipeline(True)
trans.hset(keys[1], path, json.dumps(headers).encode('utf-8'))
trans.zadd(keys[0], {path: 1}, nx=True)
trans.execute()
except Exception as err:
self.logger.error("%s: failed to create key %s (%s)", SWIFT_SOURCE,
':'.join([keys[0], path]), str(err))
raise HTTPInternalServerError()
def handle_get(self, req, start_response):
app_resp = self._app_call(req.environ)
keys = self.get_decryption_keys(req)
if keys is None:
# skip decryption
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return app_resp
mod_resp_headers = self.decrypt_resp_headers(keys)
crypto_meta = None
if is_success(self._get_status_int()):
try:
crypto_meta = self.get_crypto_meta(
'X-Object-Sysmeta-Crypto-Body-Meta')
except EncryptionException as err:
msg = 'Error decrypting object'
self.logger.error(
_('%(msg)s: %(err)s') % {
'msg': msg,
'err': err
})
raise HTTPInternalServerError(body=msg,
content_type='text/plain')
if crypto_meta:
# 2xx response and encrypted body
body_key = self.get_unwrapped_key(crypto_meta, keys['object'])
content_type, content_type_attrs = parse_content_type(
self._response_header_value('Content-Type'))
if (self._get_status_int() == 206
and content_type == 'multipart/byteranges'):
boundary = dict(content_type_attrs)["boundary"]
resp_iter = self.multipart_response_iter(
app_resp, boundary, body_key, crypto_meta)
else:
offset = 0
content_range = self._response_header_value('Content-Range')
if content_range:
# Determine offset within the whole object if ranged GET
offset, end, total = parse_content_range(content_range)
resp_iter = self.response_iter(app_resp, body_key, crypto_meta,
offset)
else:
# don't decrypt body of unencrypted or non-2xx responses
resp_iter = app_resp
mod_resp_headers = purge_crypto_sysmeta_headers(mod_resp_headers)
start_response(self._response_status, mod_resp_headers,
self._response_exc_info)
return resp_iter
def _read_crypto_meta(self, header, check):
crypto_meta = None
if (is_success(self._get_status_int())
or self._get_status_int() in (304, 412)):
try:
crypto_meta = self.get_crypto_meta(header, check)
except EncryptionException as err:
self.logger.error(_('Error decrypting object: %s'), err)
raise HTTPInternalServerError(body='Error decrypting object',
content_type='text/plain')
return crypto_meta
def get_keys(self, env, required=None, key_id=None):
# Get the key(s) from the keymaster
required = required if required is not None else [self.server_type]
try:
fetch_crypto_keys = env[CRYPTO_KEY_CALLBACK]
except KeyError:
self.logger.exception(_('ERROR get_keys() missing callback'))
raise HTTPInternalServerError(
"Unable to retrieve encryption keys.")
err = None
try:
keys = fetch_crypto_keys(key_id=key_id)
except UnknownSecretIdError as err:
self.logger.error('get_keys(): unknown key id: %s', err)
raise
except Exception as err: # noqa
self.logger.exception('get_keys(): from callback: %s', err)
raise HTTPInternalServerError(
"Unable to retrieve encryption keys.")
for name in required:
try:
key = keys[name]
self.crypto.check_key(key)
continue
except KeyError:
self.logger.exception(_("Missing key for %r") % name)
except TypeError:
self.logger.exception(_("Did not get a keys dict"))
except ValueError as e:
# don't include the key in any messages!
self.logger.exception(
_("Bad key for %(name)r: %(err)s") % {
'name': name,
'err': e
})
raise HTTPInternalServerError(
"Unable to retrieve encryption keys.")
return keys
def __call__(self, env, start_response):
req = Request(env)
new_service = env.get('liteauth.new_service', None)
if new_service:
account_id = env.get('REMOTE_USER', '')
if not account_id:
return HTTPInternalServerError()
if not self.activate_service(account_id, new_service, req.environ):
return HTTPInternalServerError()
if req.method in ['PUT', 'POST'
] and not 'x-zerovm-execute' in req.headers:
account_info = get_account_info(req.environ,
self.app,
swift_source='litequota')
service_plan = assemble_from_partial(self.metadata_key,
account_info['meta'])
if service_plan:
try:
service_plan = json.loads(service_plan)
path_parts = req.split_path(2, 4, rest_with_last=True)
except ValueError:
return self.app(env, start_response)
if len(path_parts) == 3:
quota = service_plan['storage']['containers']
new_size = int(account_info['container_count'])
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge(
body='Over quota: containers')(env, start_response)
else:
new_size = int(account_info['bytes']) + (req.content_length
or 0)
quota = service_plan['storage']['bytes']
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge(
body='Over quota: bytes')(env, start_response)
quota = service_plan['storage']['objects']
new_size = int(account_info['total_object_count'])
if 0 <= quota < new_size:
return HTTPRequestEntityTooLarge(
body='Over quota: objects')(env, start_response)
return self.app(env, start_response)
def __call__(self, env, start_response):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
replication_method = getattr(method, 'replication', False)
if (self.replication_server is not None and
self.replication_server != replication_method):
raise AttributeError('Not allowed method.')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)
except DiskFileCollision:
res = HTTPForbidden(request=req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(_(
'ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = time.time() - start_time
if self.log_requests:
log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000',
time.gmtime()),
req.method, req.path, res.status.split()[0],
res.content_length or '-', req.referer or '-',
req.headers.get('x-trans-id', '-'),
req.user_agent or '-',
trans_time)
if req.method in ('REPLICATE', 'REPLICATION') or \
'X-Backend-Replication' in req.headers:
self.logger.debug(log_line)
else:
self.logger.info(log_line)
if req.method in ('PUT', 'DELETE'):
slow = self.slow - trans_time
if slow > 0:
sleep(slow)
return res(env, start_response)
def __call__(self, env, start_response):
"""
Boilerplate code for how the server's code gets called
upon receiving a request.
Taken directly from other servers.
"""
# start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
replication_method = getattr(method, 'replication', False)
if (self.replication_server is not None
and self.replication_server != replication_method):
raise AttributeError('Not allowed method.')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(
_('ERROR __call__ error with %(method)s %(path)s '), {
'method': req.method,
'path': req.path
})
res = HTTPInternalServerError(body=traceback.format_exc())
# trans_time = '%.4f' % (time.time() - start_time)
# if self.log_requests:
# log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % (
# req.remote_addr,
# time.strftime('%d/%b/%Y:%H:%M:%S +0000',
# time.gmtime()),
# req.method, req.path,
# res.status.split()[0], res.content_length or '-',
# req.headers.get('x-trans-id', '-'),
# req.referer or '-', req.user_agent or '-',
# trans_time)
# if req.method.upper() == 'REPLICATE':
# self.logger.debug(log_message)
# else:
# self.logger.info(log_message)
return res(env, start_response)
def __call__(self, env, start_response):
print "account server __call__"
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which are not publicly accessible
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
replication_method = getattr(method, 'replication', False)
if (self.replication_server is not None
and self.replication_server != replication_method):
raise AttributeError('Not allowed method.')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(
_('ERROR __call__ error with %(method)s'
' %(path)s '), {
'method': req.method,
'path': req.path
})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = '%.4f' % (time.time() - start_time)
additional_info = ''
if res.headers.get('x-container-timestamp') is not None:
additional_info += 'x-container-timestamp: %s' % \
res.headers['x-container-timestamp']
log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()),
req.method, req.path, res.status.split()[0], res.content_length
or '-', req.headers.get('x-trans-id', '-'), req.referer
or '-', req.user_agent or '-', trans_time, additional_info)
if req.method.upper() == 'REPLICATE':
self.logger.debug(log_message)
else:
self.logger.info(log_message)
return res(env, start_response)
def __call__(self, env, start_response):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
replication_method = getattr(method, 'replication', False)
if (self.replication_server is not None and
self.replication_server != replication_method):
raise AttributeError('Not allowed method.')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)
except DiskFileCollision:
res = HTTPForbidden(request=req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(_(
'ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = time.time() - start_time
if self.log_requests:
log_line = get_log_line(req, res, trans_time, '')
if req.method in ('REPLICATE', 'REPLICATION') or \
'X-Backend-Replication' in req.headers:
self.logger.debug(log_line)
else:
self.logger.info(log_line)
if req.method in ('PUT', 'DELETE'):
slow = self.slow - trans_time
if slow > 0:
sleep(slow)
my_debug('returning from __call__ or objserver', res)
my_debug('returnign from __call__ of objserver', start_response)
return res(env, start_response)
def __call__(self, req):
try:
request_handler = self.handler_class(req, self.conf, self.app,
self.logger,
self.crystal_control)
self.logger.debug('crystal_introspection_handler call')
except:
return req.get_response(self.app)
try:
return request_handler.handle_request()
except HTTPException:
self.logger.exception('Crystal Introspection execution failed')
raise
except Exception:
self.logger.exception('Crystal Introspection execution failed')
raise HTTPInternalServerError(
body='Crystal Introspection execution failed')
def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
else:
try:
# disallow methods which are not publicly accessible
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
replication_method = getattr(method, 'replication', False)
if (self.replication_server is not None
and self.replication_server != replication_method):
raise AttributeError('Not allowed method.')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(
_('ERROR __call__ error with %(method)s'
' %(path)s '), {
'method': req.method,
'path': req.path
})
res = HTTPInternalServerError(body=traceback.format_exc())
if self.log_requests:
trans_time = time.time() - start_time
additional_info = ''
if res.headers.get('x-container-timestamp') is not None:
additional_info += 'x-container-timestamp: %s' % \
res.headers['x-container-timestamp']
log_msg = get_log_line(req, res, trans_time, additional_info)
if req.method.upper() == 'REPLICATE':
self.logger.debug(log_msg)
else:
self.logger.info(log_msg)
return res(env, start_response)
def __call__(self, env, start_response):
"""
Accepts a standard WSGI application call.
There are 2 types of requests that this middleware will affect.
1. Requests to CDN 'database' that will enable, list, etc. containers
for the CDN.
2. Requests (GETs, HEADs) from CDN provider to publicly available
containers.
The types of requests can be determined by looking at the hostname of
the incoming call.
Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
"""
env['sos.start_time'] = time()
host = env.get('HTTP_HOST', '').split(':')[0]
try:
handler = None
request_type = 'SOS_LOG'
if host in self.origin_db_hosts:
handler = OriginDbHandler(self.app, self.conf, self.logger)
request_type = 'SOS_DB'
for cdn_host_suffix in self.origin_cdn_host_suffixes:
if host.endswith(cdn_host_suffix) and \
host not in self.ignore_hosts:
handler = CdnHandler(self.app, self.conf, self.logger)
request_type = 'SOS_ORIGIN'
break
if env['PATH_INFO'].startswith(self.origin_prefix):
handler = AdminHandler(self.app, self.conf, self.logger)
request_type = 'SOS_ADMIN'
if handler:
req = Request(env)
resp = handler.handle_request(env, req)
if request_type != 'SOS_ORIGIN':
self._log_request(env, resp, request_type)
return resp(env, start_response)
except InvalidConfiguration, e:
self.logger.exception(e)
return HTTPInternalServerError(body=str(e))(env, start_response)
def __call__(self, req):
try:
if self.exec_server == 'object':
raise NotCrystalRequest
request_handler = self.handler_class(req, self.conf, self.app,
self.logger,
self.crystal_control)
self.logger.debug('crystal_metric_handler call')
except NotCrystalRequest:
return req.get_response(self.app)
try:
return request_handler.handle_request()
except HTTPException:
self.logger.exception('Crystal Metric execution failed')
raise
except Exception:
self.logger.exception('Crystal Metric execution failed')
raise HTTPInternalServerError(
body='Crystal Metric execution failed')
def get_unwrapped_key(self, crypto_meta, wrapping_key):
"""
Get a wrapped key from crypto-meta and unwrap it using the provided
wrapping key.
:param crypto_meta: a dict of crypto-meta
:param wrapping_key: key to be used to decrypt the wrapped key
:return: an unwrapped key
:raises EncryptionException: if the crypto-meta has no wrapped key or
the unwrapped key is invalid
"""
try:
return self.crypto.unwrap_key(wrapping_key,
crypto_meta['body_key'])
except KeyError as err:
err = 'Missing %s' % err
except ValueError as err:
pass
msg = 'Error decrypting %s' % self.server_type
self.logger.error(_('%(msg)s: %(err)s') % {'msg': msg, 'err': err})
raise HTTPInternalServerError(body=msg, content_type='text/plain')
def handle_request(self, env, req):
"""
This handles requests from a user to activate cdn access for their
containers, list them, etc.
"""
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
try:
if req.method in ('PUT', 'POST'):
return self.origin_db_puts_posts(env, req)
if req.method == 'GET':
return self.origin_db_get(env, req)
if req.method == 'HEAD':
return self.origin_db_head(env, req)
if req.method == 'DELETE':
return self.origin_db_delete(env, req)
except OriginDbFailure, e:
self.logger.exception(e)
return HTTPInternalServerError('Origin DB Failure')
def __call__(self, req):
try:
request_handler = self.handler_class(req, self.vertigo_conf,
self.app, self.logger)
self.logger.debug(
'vertigo_handler %s call in %s with %s/%s/%s' %
(req.method, self.exec_server, request_handler.account,
request_handler.container, request_handler.obj))
except HTTPException:
raise
except NotVertigoRequest:
return req.get_response(self.app)
try:
return request_handler.handle_request()
except HTTPException:
self.logger.exception('Vertigo execution failed')
raise
except Exception:
self.logger.exception('Vertigo execution failed')
raise HTTPInternalServerError(body='Vertigo execution failed')
def __call__(self, req):
try:
request_handler = self.handler_class(req, self.conf, self.app,
self.logger)
self.logger.debug(
'%s call in %s-server with %s/%s/%s' %
(req.method, self.exec_server, request_handler.account,
request_handler.container, request_handler.obj))
except HTTPException:
raise
except NotCrystalRequest:
return req.get_response(self.app)
try:
return request_handler.handle_request()
except HTTPException:
self.logger.exception('Middleware execution failed')
raise
except Exception:
self.logger.exception('Middleware execution failed')
raise HTTPInternalServerError(
body='Crystal filter middleware execution failed')
def __call__(self, req):
"""Accept a standard WSGI app call.
The call takes one of two paths:
- Handle a request to the auth system (e.g. creating
new access keys or changing a secret key).
- Authenticate a request which is signed with a s3 signature.
"""
try:
if req.path_info.startswith(self.auth_prefix):
return self.handle_auth_api(req)
if 'swift3.auth_details' in req.environ:
auth_details = req.environ['swift3.auth_details']
akey = auth_details['access_key']
secret, account = self._get_details(req, akey)
if secret:
# Authentication.
if auth_details['check_signature'](secret.encode('utf-8')):
req.environ['swift.authorize_override'] = True
# Authorization function (used later in pipeline).
req.environ['swift.authorize'] = self._authorize
req.remote_user = account
# swift3 sets account to access_key . Replace.
req.environ['PATH_INFO'] = req.environ['PATH_INFO'].\
replace(akey, account, 1)
else:
return _denied_response(req)
else:
return _denied_response(req)
except Exception:
self.logger.error('EXCEPTION occured: %s: %s', format_exc(),
req.environ)
return HTTPInternalServerError(request=req)
return self.app
def __call__(self, req):
try:
r = redis.Redis(connection_pool=self.redis_conn_pool)
handler = self.handler_class(req, self.conf, self.app, self.logger,
r)
self.logger.debug('%s call in %s' % (req.method, req.path))
return handler.handle_request()
except NotFunctionRequest:
self.logger.debug('No Zion Request, bypassing middleware')
return req.get_response(self.app)
except HTTPException:
self.logger.exception('Zion execution failed')
raise
except Exception:
self.logger.exception('Zion execution failed')
raise HTTPInternalServerError(
body=
'Unable to execute the requested operation: Zion execution failed\n'
)
def _create_placeholder(self, req, account, container, mode, path):
"""
Create a "placeholder" telling that a container has been created
to hold objects sharing a common prefix.
:param mode: OBJ or CNT.
"""
key = self.key(account, container, mode, path)
LOG.debug("%s: create key %s", self.SWIFT_SOURCE, key)
# TODO: should we increase number of objects ?
# but we should manage in this case all error case
# to avoid false counter
try:
if self.redis_keys_format == REDIS_KEYS_FORMAT_V1:
self.conn.setnx(key, "1")
elif self.redis_keys_format == REDIS_KEYS_FORMAT_V2:
self.conn.hsetnx(key, path, "1")
else:
self.conn.zset(key, path)
except Exception as e:
LOG.error("%s: failed to create key %s (%s)", self.SWIFT_SOURCE,
':'.join([key, path]), str(e))
raise HTTPInternalServerError()
def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(wsgi_to_str(req.path_info), internal=True):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
# disallow methods which are not publicly accessible
if req.method not in self.allowed_methods:
res = HTTPMethodNotAllowed()
else:
res = getattr(self, req.method)(req)
except HTTPException as error_response:
res = error_response
except (Exception, Timeout):
self.logger.exception(
_('ERROR __call__ error with %(method)s'
' %(path)s '), {
'method': req.method,
'path': req.path
})
res = HTTPInternalServerError(body=traceback.format_exc())
if self.log_requests:
trans_time = time.time() - start_time
additional_info = ''
if res.headers.get('x-container-timestamp') is not None:
additional_info += 'x-container-timestamp: %s' % \
res.headers['x-container-timestamp']
log_msg = get_log_line(req, res, trans_time, additional_info,
self.log_format, self.anonymization_method,
self.anonymization_salt)
if req.method.upper() == 'REPLICATE':
self.logger.debug(log_msg)
else:
self.logger.info(log_msg)
return res(env, start_response)
def _store_object(self, req, data_source, headers):
kwargs = {}
content_type = req.headers.get('content-type', 'octet/stream')
policy = None
container_info = self.container_info(self.account_name,
self.container_name, req)
if 'X-Oio-Storage-Policy' in req.headers:
policy = req.headers.get('X-Oio-Storage-Policy')
if not self.app.POLICIES.get_by_name(policy):
raise HTTPBadRequest(
"invalid policy '%s', must be in %s" %
(policy, self.app.POLICIES.by_name.keys()))
else:
try:
policy_index = int(
req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy']))
except TypeError:
policy_index = 0
if policy_index != 0:
policy = self.app.POLICIES.get_by_index(policy_index).name
else:
content_length = int(req.headers.get('content-length', 0))
policy = self._get_auto_policy_from_size(content_length)
ct_props = {'properties': {}, 'system': {}}
metadata = self.load_object_metadata(headers)
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('swift.perfdata')
# only send headers if needed
if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER):
oio_headers[FORCEVERSIONING_HEADER] = \
headers.get(FORCEVERSIONING_HEADER)
if req.environ.get('oio.force-version'):
# In a case of MPU, it contains version of the UploadId
# to be able to include version-id of MPU in S3 reponse
kwargs['version'] = req.environ.get('oio.force-version')
# In case a shard is being created, save the name of the S3 bucket
# in a container property. This will be used when aggregating
# container statistics to make bucket statistics.
if BUCKET_NAME_HEADER in headers:
bname = headers[BUCKET_NAME_HEADER]
# FIXME(FVE): the segments container is not part of another bucket!
# We should not have to strip this here.
if bname and bname.endswith(MULTIUPLOAD_SUFFIX):
bname = bname[:-len(MULTIUPLOAD_SUFFIX)]
ct_props['system'][BUCKET_NAME_PROP] = bname
try:
_chunks, _size, checksum, _meta = self._object_create(
self.account_name,
self.container_name,
obj_name=self.object_name,
file_or_path=data_source,
mime_type=content_type,
policy=policy,
headers=oio_headers,
etag=req.headers.get('etag', '').strip('"'),
properties=metadata,
container_properties=ct_props,
cache=oio_cache,
perfdata=perfdata,
**kwargs)
# TODO(FVE): when oio-sds supports it, do that in a callback
# passed to object_create (or whatever upload method supports it)
footer_md = self.load_object_metadata(self._get_footers(req))
if footer_md:
self.app.storage.object_set_properties(self.account_name,
self.container_name,
self.object_name,
version=_meta.get(
'version', None),
properties=footer_md,
headers=oio_headers,
cache=oio_cache,
perfdata=perfdata)
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except SourceReadTimeout as err:
self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
self.app.logger.increment('client_timeouts')
raise HTTPRequestTimeout(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout,
exceptions.DeadlineReached):
raise
except exceptions.NoSuchContainer:
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
last_modified = int(_meta.get('mtime', math.ceil(time.time())))
# FIXME(FVE): if \x10 character in object name, decode version
# number and set it in the response headers, instead of the oio
# version number.
version_id = _meta.get('version', 'null')
resp = HTTPCreated(request=req,
etag=checksum,
last_modified=last_modified,
headers={'x-object-sysmeta-version-id': version_id})
return resp
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info(
"Creating link from %s/%s/%s to %s/%s/%s",
# Existing
from_account,
container,
obj,
# New
self.account_name,
self.container_name,
self.object_name)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('swift.perfdata')
# FIXME(FVE): use object_show, cache in req.environ
version = obj_version_from_env(req.environ)
props = storage.object_get_properties(from_account,
container,
obj,
headers=oio_headers,
version=version,
cache=oio_cache,
perfdata=perfdata)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
else:
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
link_meta = storage.object_link(from_account,
container,
obj,
self.account_name,
self.container_name,
self.object_name,
headers=oio_headers,
properties=metadata,
properties_directive='REPLACE',
target_version=version,
cache=oio_cache,
perfdata=perfdata)
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout,
exceptions.DeadlineReached):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=link_meta['hash'])
return resp
def __call__(self, req):
if not self.super_admin_key: # profile management is disabled
return self.denied_response(req)
try:
(endpoint, _rest) = req.split_path(1, 2, True)
except ValueError:
return self.denied_response(req)
if endpoint == self.profile_path:
account_id = req.environ.get('REMOTE_USER', '')
if not account_id:
return HTTPUnauthorized(request=req)
user_id, user_email = account_id.split(':')
whitelist_id = get_data_from_url(self.whitelist_url,
self.app,
user_email,
self.logger,
req.environ)
invite_code = req.headers.get('x-auth-invite-code', None)
if invite_code and req.method == 'PUT':
invite_id = get_data_from_url(self.invite_url,
self.app,
invite_code,
self.logger,
req.environ)
if not invite_id:
return self.denied_response(req)
service = None
if not invite_id.startswith('email:'):
if not store_data_in_url(self.invite_url,
self.app,
invite_code,
'email:%s:%s' % (user_email, invite_id),
req.environ):
return HTTPInternalServerError(request=req)
service = invite_id
elif 'email:%s:' % user_email in invite_id:
service = invite_id.split(':', 3)[2]
if service and not whitelist_id:
if not store_data_in_url(self.whitelist_url,
self.app,
user_email,
service,
req.environ):
return HTTPInternalServerError(request=req)
whitelist_id = service
if not whitelist_id:
return Response(request=req, status=402,
body='Account not in whitelist')
if whitelist_id.startswith('service_'):
req.environ['liteauth.new_service'] = \
whitelist_id.replace('service_', '', 1)
if not store_data_in_url(self.whitelist_url,
self.app,
user_email,
user_id,
req.environ):
return HTTPInternalServerError(request=req)
if req.method == 'GET':
return self.get_swauth(req, user_id, user_email)
elif req.method == 'PUT':
return self.put_swauth(req, user_id, user_email)
return self.denied_response(req)
