def POST(self, req):
"""HTTP POST request handler."""
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
if not req.environ.get('swift_owner'):
for key in self.app.swift_owner_headers:
req.headers.pop(key, None)
if req.environ.get('reseller_request', False) and \
'X-Container-Sharding' in req.headers:
req.headers[get_sys_meta_prefix('container') + 'Sharding'] = \
str(config_true_value(req.headers['X-Container-Sharding']))
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self.generate_request_headers(req, transfer=True)
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
req, self.app.container_ring, container_partition, 'POST',
req.swift_entity_path, [headers] * len(containers))
return resp
def extract_acl_and_report_errors(self, req):
"""
Return a user-readable string indicating the errors in the input ACL,
or None if there are no errors.
"""
acl_header = 'x-account-access-control'
acl_data = req.headers.get(acl_header)
result = parse_acl(version=2, data=acl_data)
if result is None:
return 'Syntax error in input (%r)' % acl_data
tempauth_acl_keys = 'admin read-write read-only'.split()
for key in result:
# While it is possible to construct auth systems that collaborate
# on ACLs, TempAuth is not such an auth system. At this point,
# it thinks it is authoritative.
if key not in tempauth_acl_keys:
return 'Key %r not recognized' % key
for key in tempauth_acl_keys:
if key not in result:
continue
if not isinstance(result[key], list):
return 'Value for key %r must be a list' % key
for grantee in result[key]:
if not isinstance(grantee, str):
return 'Elements of %r list must be strings' % key
# Everything looks fine, no errors found
internal_hdr = get_sys_meta_prefix('account') + 'core-access-control'
req.headers[internal_hdr] = req.headers.pop(acl_header)
return None
def client_to_sysmeta(self, req, req_type):
subresources = {
'account': ('container', 'object'),
'container': ('object', ),
}.get(req_type, ())
header_formats = (
('x-remove-default-%s-', True),
('x-default-%s-', False),
)
for header_format, clear in header_formats:
for header, value in req.headers.items():
for subresource in subresources:
prefix = header_format % subresource
if header.lower().startswith(prefix):
header_to_default = header[len(prefix):].lower()
if header_to_default.startswith(BLACKLIST_PREFIXES):
continue
if header_to_default in BLACKLIST:
continue
sysmeta_header = '%sdefault-%s-%s' % (
get_sys_meta_prefix(req_type),
subresource,
header_to_default)
req.headers[sysmeta_header] = '' if clear else value
def test_headers_to_object_info_sys_meta(self):
prefix = get_sys_meta_prefix("object")
headers = {"%sWhatevs" % prefix: 14, "%ssomethingelse" % prefix: 0}
resp = headers_to_object_info(headers.items(), 200)
self.assertEquals(len(resp["sysmeta"]), 2)
self.assertEquals(resp["sysmeta"]["whatevs"], 14)
self.assertEquals(resp["sysmeta"]["somethingelse"], 0)
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
ai = self.account_info(self.account_name, req)
auto_account = self.account_name.startswith(
self.app.auto_create_account_prefix)
if not (auto_account or ai[1]):
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
# Don't cache this. The lack of account will be cached, and that
# is sufficient.
return HTTPNotFound(request=req)
part = self.app.container_ring.get_part(
self.account_name, self.container_name)
concurrency = self.app.container_ring.replica_count \
if self.app.concurrent_gets else 1
node_iter = self.app.iter_nodes(self.app.container_ring, part)
params = req.params
params['format'] = 'json'
record_type = req.headers.get('X-Backend-Record-Type', '').lower()
if not record_type:
record_type = 'auto'
req.headers['X-Backend-Record-Type'] = 'auto'
params['states'] = 'listing'
req.params = params
resp = self.GETorHEAD_base(
req, _('Container'), node_iter, part,
req.swift_entity_path, concurrency)
resp_record_type = resp.headers.get('X-Backend-Record-Type', '')
if all((req.method == "GET", record_type == 'auto',
resp_record_type.lower() == 'shard')):
resp = self._get_from_shards(req, resp)
# Cache this. We just made a request to a storage node and got
# up-to-date information for the container.
resp.headers['X-Backend-Recheck-Container-Existence'] = str(
self.app.recheck_container_existence)
set_info_cache(self.app, req.environ, self.account_name,
self.container_name, resp)
if 'swift.authorize' in req.environ:
req.acl = resp.headers.get('x-container-read')
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
if not req.environ.get('swift_owner', False):
for key in self.app.swift_owner_headers:
if key in resp.headers:
del resp.headers[key]
# Expose sharding state in reseller requests
if req.environ.get('reseller_request', False):
resp.headers['X-Container-Sharding'] = config_true_value(
resp.headers.get(get_sys_meta_prefix('container') + 'Sharding',
'False'))
return resp
def test_headers_to_object_info_sys_meta(self):
prefix = get_sys_meta_prefix('object')
headers = {'%sWhatevs' % prefix: 14,
'%ssomethingelse' % prefix: 0}
resp = headers_to_object_info(headers.items(), 200)
self.assertEqual(len(resp['sysmeta']), 2)
self.assertEqual(resp['sysmeta']['whatevs'], 14)
self.assertEqual(resp['sysmeta']['somethingelse'], 0)
def add_acls_from_sys_metadata(self, resp):
if resp.environ["REQUEST_METHOD"] in ("HEAD", "GET", "PUT", "POST"):
prefix = get_sys_meta_prefix("account") + "core-"
name = "access-control"
(extname, intname) = ("x-account-" + name, prefix + name)
acl_dict = parse_acl(version=2, data=resp.headers.pop(intname))
if acl_dict: # treat empty dict as empty header
resp.headers[extname] = format_acl(version=2, acl_dict=acl_dict)
def add_acls_from_sys_metadata(self, resp):
if resp.environ['REQUEST_METHOD'] in ('HEAD', 'GET', 'PUT', 'POST'):
prefix = get_sys_meta_prefix('account') + 'core-'
name = 'access-control'
(extname, intname) = ('x-account-' + name, prefix + name)
acl_dict = parse_acl(version=2, data=resp.headers.pop(intname))
if acl_dict: # treat empty dict as empty header
resp.headers[extname] = format_acl(
version=2, acl_dict=acl_dict)
def _make_user_and_sys_acl_headers_data(self):
acl = {
'admin': ['AUTH_alice', 'AUTH_bob'],
'read-write': ['AUTH_carol'],
'read-only': [],
}
user_prefix = 'x-account-' # external, user-facing
user_headers = {(user_prefix + 'access-control'): format_acl(
version=2, acl_dict=acl)}
sys_prefix = get_sys_meta_prefix('account') # internal, system-facing
sys_headers = {(sys_prefix + 'core-access-control'): format_acl(
version=2, acl_dict=acl)}
return user_headers, sys_headers
def PUT(self, req):
"""HTTP PUT request handler."""
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
policy_index = self._convert_policy_to_index(req)
if not req.environ.get('swift_owner'):
for key in self.app.swift_owner_headers:
req.headers.pop(key, None)
if req.environ.get('reseller_request', False) and \
'X-Container-Sharding' in req.headers:
req.headers[get_sys_meta_prefix('container') + 'Sharding'] = \
str(config_true_value(req.headers['X-Container-Sharding']))
length_limit = self.get_name_length_limit()
if len(self.container_name) > length_limit:
body = 'Container name length of %d longer than %d' % (
len(self.container_name), length_limit)
resp = HTTPBadRequest(request=req, body=body)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts and self.app.account_autocreate:
if not self.autocreate_account(req, self.account_name):
return HTTPServiceUnavailable(request=req)
account_partition, accounts, container_count = \
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
if 0 < self.app.max_containers_per_account <= container_count and \
self.account_name not in self.app.max_containers_whitelist:
container_info = \
self.container_info(self.account_name, self.container_name,
req)
if not is_success(container_info.get('status')):
body = 'Reached container limit of %s' % (
self.app.max_containers_per_account, )
resp = HTTPForbidden(request=req, body=body)
return resp
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts,
policy_index)
resp = self.make_requests(
req, self.app.container_ring,
container_partition, 'PUT', req.swift_entity_path, headers)
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
return resp
def test_sys_meta_headers_PUT(self):
sys_meta_key = '%stest' % get_sys_meta_prefix('container')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Container-Meta-Test'
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a/c', headers=hdrs_in, method='PUT')
self.storage.container.container_create = Mock()
req.get_response(self.app)
meta = \
self.storage.container.container_create.call_args[1]['properties']
self.assertEqual(meta[sys_meta_key], 'foo')
self.assertEqual(meta[user_meta_key], 'bar')
def test_sys_meta_headers_POST(self):
# check that headers in sys meta namespace make it through
# the proxy controller
sys_meta_key = '%stest' % get_sys_meta_prefix('account')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Account-Meta-Test'
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a', headers=hdrs_in, method='POST')
self.storage.account_set_properties = Mock()
req.get_response(self.app)
meta = self.storage.account_set_properties.call_args[1]['properties']
self.assertEqual(meta[sys_meta_key], 'foo')
self.assertEqual(meta[user_meta_key], 'bar')
def test_sys_meta_headers_POST(self):
# check that headers in sys meta namespace make it through
# the container controller
sys_meta_key = '%stest' % get_sys_meta_prefix('container')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Container-Meta-Test'
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a/c', headers=hdrs_in, method='POST')
self.storage.container.container_set_properties = Mock(
return_value="")
req.get_response(self.app)
meta = self.storage.container.container_set_properties.call_args[0][2]
self.assertEqual(meta[sys_meta_key], 'foo')
self.assertEqual(meta[user_meta_key], 'bar')
def handle_container_request(self, env, start_response):
app_resp = self._app_call(env)
if self._response_headers is None:
self._response_headers = []
sysmeta_version_hdr = get_sys_meta_prefix('container') + \
'versions-location'
location = ''
for key, val in self._response_headers:
if key.lower() == sysmeta_version_hdr:
location = val
if location:
self._response_headers.extend([('X-Versions-Location', location)])
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return app_resp
def container_request(self, req, start_response, enabled):
sysmeta_version_hdr = get_sys_meta_prefix('container') + \
'versions-location'
# set version location header as sysmeta
if 'X-Versions-Location' in req.headers:
val = req.headers.get('X-Versions-Location')
if val:
# differently from previous version, we are actually
# returning an error if user tries to set versions location
# while feature is explicitly disabled.
if not config_true_value(enabled) and \
req.method in ('PUT', 'POST'):
raise HTTPPreconditionFailed(
request=req, content_type='text/plain',
body='Versioned Writes is disabled')
location = check_container_format(req, val)
req.headers[sysmeta_version_hdr] = location
# reset original header to maintain sanity
# now only sysmeta is source of Versions Location
req.headers['X-Versions-Location'] = ''
# if both headers are in the same request
# adding location takes precedence over removing
if 'X-Remove-Versions-Location' in req.headers:
del req.headers['X-Remove-Versions-Location']
else:
# empty value is the same as X-Remove-Versions-Location
req.headers['X-Remove-Versions-Location'] = 'x'
# handle removing versions container
val = req.headers.get('X-Remove-Versions-Location')
if val:
req.headers.update({sysmeta_version_hdr: ''})
req.headers.update({'X-Versions-Location': ''})
del req.headers['X-Remove-Versions-Location']
# send request and translate sysmeta headers from response
vw_ctx = VersionedWritesContext(self.app, self.logger)
return vw_ctx.handle_container_request(req.environ, start_response)
def test_sys_meta_headers_PUT(self):
# check that headers in sys meta namespace make it through
# the proxy controller
sys_meta_key = '%stest' % get_sys_meta_prefix('account')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Account-Meta-Test'
# allow PUTs to account...
self.app.allow_account_management = True
hdrs_in = {
sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'
}
req = Request.blank('/v1/a', headers=hdrs_in, method='PUT')
self.storage.account_create = Mock()
self.storage.account_set_properties = Mock()
req.get_response(self.app)
meta = self.storage.account_set_properties.call_args[0][1]
self.assertEqual(meta[sys_meta_key], 'foo')
self.assertEqual(meta[user_meta_key], 'bar')
def test_sys_meta_headers_POST(self):
# check that headers in sys meta namespace make it through
# the container controller
sys_meta_key = "%stest" % get_sys_meta_prefix("container")
sys_meta_key = sys_meta_key.title()
user_meta_key = "X-Container-Meta-Test"
controller = proxy_server.ContainerController(self.app, "a", "c")
context = {}
callback = self._make_callback_func(context)
hdrs_in = {sys_meta_key: "foo", user_meta_key: "bar", "x-timestamp": "1.0"}
req = Request.blank("/v1/a/c", headers=hdrs_in)
with mock.patch(
"swift.proxy.controllers.base.http_connect", fake_http_connect(200, 200, give_connect=callback)
):
controller.POST(req)
self.assertEqual(context["method"], "POST")
self.assertTrue(sys_meta_key in context["headers"])
self.assertEqual(context["headers"][sys_meta_key], "foo")
self.assertTrue(user_meta_key in context["headers"])
self.assertEqual(context["headers"][user_meta_key], "bar")
self.assertNotEqual(context["headers"]["x-timestamp"], "1.0")
def test_sys_meta_headers_POST(self):
# check that headers in sys meta namespace make it through
# the proxy controller
sys_meta_key = '%stest' % get_sys_meta_prefix('account')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Account-Meta-Test'
controller = proxy_server.AccountController(self.app, 'a')
context = {}
callback = self._make_callback_func(context)
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a', headers=hdrs_in)
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, give_connect=callback)):
controller.POST(req)
self.assertEqual(context['method'], 'POST')
self.assertTrue(sys_meta_key in context['headers'])
self.assertEqual(context['headers'][sys_meta_key], 'foo')
self.assertTrue(user_meta_key in context['headers'])
self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
def test_sys_meta_headers_POST(self):
# check that headers in sys meta namespace make it through
# the proxy controller
sys_meta_key = '%stest' % get_sys_meta_prefix('account')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Account-Meta-Test'
controller = proxy_server.AccountController(self.app, 'a')
context = {}
callback = self._make_callback_func(context)
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a', headers=hdrs_in)
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, give_connect=callback)):
controller.POST(req)
self.assertEqual(context['method'], 'POST')
self.assertTrue(sys_meta_key in context['headers'])
self.assertEqual(context['headers'][sys_meta_key], 'foo')
self.assertTrue(user_meta_key in context['headers'])
self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
def client_to_sysmeta(self, req, req_type):
subresources = {
'account': ('container', 'object'),
'container': ('object', ),
}.get(req_type, ())
header_formats = (
('x-remove-default-%s-', True),
('x-default-%s-', False),
)
for header_format, clear in header_formats:
for header, value in req.headers.items():
for subresource in subresources:
prefix = header_format % subresource
if header.lower().startswith(prefix):
header_to_default = header[len(prefix):].lower()
if header_to_default.startswith(BLACKLIST_PREFIXES):
continue
if header_to_default in BLACKLIST:
continue
sysmeta_header = '%sdefault-%s-%s' % (
get_sys_meta_prefix(req_type), subresource,
header_to_default)
req.headers[sysmeta_header] = '' if clear else value
"""
from swift.common.swob import Request
from swift.common.utils import get_logger
from swift.common.request_helpers import remove_items, get_sys_meta_prefix
import re
#: A list of python regular expressions that will be used to
#: match against inbound request headers. Matching headers will
#: be removed from the request.
# Exclude headers starting with a sysmeta prefix.
# If adding to this list, note that these are regex patterns,
# so use a trailing $ to constrain to an exact header match
# rather than prefix match.
inbound_exclusions = [get_sys_meta_prefix('account'),
get_sys_meta_prefix('container'),
get_sys_meta_prefix('object'),
'x-backend']
# 'x-object-sysmeta' is reserved in anticipation of future support
# for system metadata being applied to objects
#: A list of python regular expressions that will be used to
#: match against outbound response headers. Matching headers will
#: be removed from the response.
outbound_exclusions = inbound_exclusions
def make_exclusion_test(exclusions):
expr = '|'.join(exclusions)
def _save_bucket_name(self, env):
req = Request(env)
account, container, obj = self._extract_path(req.path_info)
sys_meta_key = '%soio-bucket-name' % get_sys_meta_prefix('object')
req.headers[sys_meta_key] = container
swift.defaulter_hook
This is a callback that may be used to populate defaults for subrequests.
It will only modify PUT requests. It accepts a swob.Request as an argument.
"""
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.swob import wsgify
from swift.common.utils import config_true_value
from swift.common.utils import register_swift_info
from swift.proxy.controllers.base import get_account_info
from swift.proxy.controllers.base import get_container_info
BLACKLIST = set('x-timestamp')
BLACKLIST_PREFIXES = (
get_sys_meta_prefix('account'),
get_sys_meta_prefix('container'),
get_sys_meta_prefix('object'),
'x-backend-',
)
CALLBACK_ENV_KEY = 'swift.defaulter_hook'
HEADERS_ENV_KEY = 'swift.defaulter_headers'
class DefaulterMiddleware(object):
def __init__(self, app, config):
self.app = app
self.conf = config
@wsgify
def __call__(self, req):
FileLikeIter,
)
from swift.common.request_helpers import get_sys_meta_prefix, copy_header_subset
from swift.common.wsgi import WSGIContext, make_pre_authed_request
from swift.common.swob import Request, HTTPException, HTTPRequestEntityTooLarge
from swift.common.constraints import check_container_format, MAX_FILE_SIZE
from swift.proxy.controllers.base import get_container_info
from swift.common.http import is_success, is_client_error, HTTP_NOT_FOUND
from swift.common.swob import HTTPPreconditionFailed, HTTPServiceUnavailable, HTTPServerError, HTTPBadRequest
from swift.common.exceptions import ListingIterNotFound, ListingIterError
VERSIONING_MODES = ("stack", "history")
DELETE_MARKER_CONTENT_TYPE = "application/x-deleted;swift_versions_deleted=1"
VERSIONS_LOC_CLIENT = "x-versions-location"
VERSIONS_LOC_SYSMETA = get_sys_meta_prefix("container") + "versions-location"
VERSIONS_MODE_CLIENT = "x-versions-mode"
VERSIONS_MODE_SYSMETA = get_sys_meta_prefix("container") + "versions-mode"
class VersionedWritesContext(WSGIContext):
def __init__(self, wsgi_app, logger):
WSGIContext.__init__(self, wsgi_app)
self.logger = logger
def _listing_iter(self, account_name, lcontainer, lprefix, req):
try:
for page in self._listing_pages_iter(account_name, lcontainer, lprefix, req.environ):
for item in page:
yield item
except ListingIterNotFound:
MD5_OF_EMPTY_STRING, closing_if_possible, quote
from swift.common.constraints import check_account_format
from swift.common.wsgi import WSGIContext, make_subrequest
from swift.common.request_helpers import get_sys_meta_prefix, \
check_path_header
from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \
HTTPException, HTTPConflict, HTTPPreconditionFailed
from swift.common.http import is_success
from swift.common.exceptions import LinkIterError
from swift.common.header_key_dict import HeaderKeyDict
DEFAULT_SYMLOOP_MAX = 2
# Header values for symlink target path strings will be quoted values.
TGT_OBJ_SYMLINK_HDR = 'x-symlink-target'
TGT_ACCT_SYMLINK_HDR = 'x-symlink-target-account'
TGT_OBJ_SYSMETA_SYMLINK_HDR = get_sys_meta_prefix('object') + 'symlink-target'
TGT_ACCT_SYSMETA_SYMLINK_HDR = \
get_sys_meta_prefix('object') + 'symlink-target-account'
def _check_symlink_header(req):
"""
Validate that the value from x-symlink-target header is
well formatted. We assume the caller ensures that
x-symlink-target header is present in req.headers.
:param req: HTTP request object
:raise: HTTPPreconditionFailed if x-symlink-target value
is not well formatted.
:raise: HTTPBadRequest if the x-symlink-target value points to the request
path.
get_sys_meta_prefix, update_etag_is_at_header
from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success
from swift.common.wsgi import WSGIContext, make_subrequest
from swift.common.middleware.bulk import get_response_body, \
ACCEPTABLE_FORMATS, Bulk
DEFAULT_RATE_LIMIT_UNDER_SIZE = 1024 * 1024 # 1 MiB
DEFAULT_MAX_MANIFEST_SEGMENTS = 1000
DEFAULT_MAX_MANIFEST_SIZE = 1024 * 1024 * 2 # 2 MiB
REQUIRED_SLO_KEYS = set(['path', 'etag', 'size_bytes'])
OPTIONAL_SLO_KEYS = set(['range'])
ALLOWED_SLO_KEYS = REQUIRED_SLO_KEYS | OPTIONAL_SLO_KEYS
SYSMETA_SLO_ETAG = get_sys_meta_prefix('object') + 'slo-etag'
SYSMETA_SLO_SIZE = get_sys_meta_prefix('object') + 'slo-size'
def parse_and_validate_input(req_body, req_path):
"""
Given a request body, parses it and returns a list of dictionaries.
The output structure is nearly the same as the input structure, but it
is not an exact copy. Given a valid input dictionary `d_in`, its
corresponding output dictionary `d_out` will be as follows:
* d_out['etag'] == d_in['etag']
* d_out['path'] == d_in['path']
HOP_BY_HOP_HEADERS = set([
'connection',
'keep-alive',
'proxy-authenticate',
'proxy-authorization',
'te',
'trailer',
'transfer-encoding',
'upgrade',
])
PROPAGATED_HDRS = ['x-container-read', 'x-container-write']
MIGRATOR_HEADER = 'multi-cloud-internal-migrator'
SHUNT_BYPASS_HEADER = 'x-cloud-sync-shunt-bypass'
ACCOUNT_ACL_KEY = 'x-account-access-control'
SYSMETA_ACCOUNT_ACL_KEY = \
get_sys_meta_prefix('account') + 'core-access-control'
class MigrationContainerStates(object):
'''Possible states of the migrated containers.
When a container is migrated, we have to handle the possible state
transitions it could go through. Initially, a container only exists in the
source cluster. Once we create it in the destination cluster, we tag it to
be in the MIGRATING state. In this state, we process any new objects or
updated metadata. When an object is removed, we will also remove it from
the destination container.
When a container is removed on the source and the destination container is
in the MIGRATING state, it will be removed.
from swift.common.request_helpers import get_sys_meta_prefix, \
copy_header_subset
from swift.common.wsgi import WSGIContext, make_pre_authed_request
from swift.common.swob import (Request, HTTPException,
HTTPRequestEntityTooLarge)
from swift.common.constraints import check_container_format, MAX_FILE_SIZE
from swift.proxy.controllers.base import get_container_info
from swift.common.http import (is_success, is_client_error, HTTP_NOT_FOUND)
from swift.common.swob import HTTPPreconditionFailed, HTTPServiceUnavailable, \
HTTPServerError, HTTPBadRequest
from swift.common.exceptions import (ListingIterNotFound, ListingIterError)
DELETE_MARKER_CONTENT_TYPE = 'application/x-deleted;swift_versions_deleted=1'
CLIENT_VERSIONS_LOC = 'x-versions-location'
CLIENT_HISTORY_LOC = 'x-history-location'
SYSMETA_VERSIONS_LOC = get_sys_meta_prefix('container') + 'versions-location'
SYSMETA_VERSIONS_MODE = get_sys_meta_prefix('container') + 'versions-mode'
class VersionedWritesContext(WSGIContext):
def __init__(self, wsgi_app, logger):
WSGIContext.__init__(self, wsgi_app)
self.logger = logger
def _listing_iter(self, account_name, lcontainer, lprefix, req):
try:
for page in self._listing_pages_iter(account_name, lcontainer,
lprefix, req.environ):
for item in page:
yield item
except ListingIterNotFound:
from swift.common.swob import (
Request, HTTPException, HTTPRequestEntityTooLarge)
from swift.common.constraints import check_container_format, MAX_FILE_SIZE
from swift.proxy.controllers.base import get_container_info
from swift.common.http import (
is_success, is_client_error, HTTP_NOT_FOUND)
from swift.common.swob import HTTPPreconditionFailed, HTTPServiceUnavailable, \
HTTPServerError, HTTPBadRequest
from swift.common.exceptions import (
ListingIterNotFound, ListingIterError)
DELETE_MARKER_CONTENT_TYPE = 'application/x-deleted;swift_versions_deleted=1'
CLIENT_VERSIONS_LOC = 'x-versions-location'
CLIENT_HISTORY_LOC = 'x-history-location'
SYSMETA_VERSIONS_LOC = get_sys_meta_prefix('container') + 'versions-location'
SYSMETA_VERSIONS_MODE = get_sys_meta_prefix('container') + 'versions-mode'
class VersionedWritesContext(WSGIContext):
def __init__(self, wsgi_app, logger):
WSGIContext.__init__(self, wsgi_app)
self.logger = logger
def _listing_iter(self, account_name, lcontainer, lprefix, req):
try:
for page in self._listing_pages_iter(account_name, lcontainer,
lprefix, req.environ):
for item in page:
yield item
def get_sys_migrator_header(path_type):
if path_type == 'object':
return get_object_transient_sysmeta(MIGRATOR_HEADER)
return '%s%s' % (get_sys_meta_prefix(path_type), MIGRATOR_HEADER)
def sysmeta_to_client(self, resp, req_type):
prefix = get_sys_meta_prefix(req_type) + 'default-'
for header, value in resp.headers.items():
if header.lower().startswith(prefix):
client_header = 'x-default-%s' % header[len(prefix):]
resp.headers[client_header] = value
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success
from swift.common.wsgi import WSGIContext, make_subrequest
from swift.common.middleware.bulk import get_response_body, \
ACCEPTABLE_FORMATS, Bulk
DEFAULT_RATE_LIMIT_UNDER_SIZE = 1024 * 1024 # 1 MiB
DEFAULT_MAX_MANIFEST_SEGMENTS = 1000
DEFAULT_MAX_MANIFEST_SIZE = 1024 * 1024 * 2 # 2 MiB
REQUIRED_SLO_KEYS = set(['path'])
OPTIONAL_SLO_KEYS = set(['range', 'etag', 'size_bytes'])
ALLOWED_SLO_KEYS = REQUIRED_SLO_KEYS | OPTIONAL_SLO_KEYS
SYSMETA_SLO_ETAG = get_sys_meta_prefix('object') + 'slo-etag'
SYSMETA_SLO_SIZE = get_sys_meta_prefix('object') + 'slo-size'
def parse_and_validate_input(req_body, req_path):
"""
Given a request body, parses it and returns a list of dictionaries.
The output structure is nearly the same as the input structure, but it
is not an exact copy. Given a valid input dictionary `d_in`, its
corresponding output dictionary `d_out` will be as follows:
* d_out['etag'] == d_in['etag']
* d_out['path'] == d_in['path']
def _save_bucket_name(self, env):
req = Request(env)
account, container, obj = self._extract_path(req.path_info)
sys_meta_key = '%soio-bucket-name' % get_sys_meta_prefix('object')
req.headers[sys_meta_key] = container
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from swift.common import utils as swift_utils
from swift.common.http import is_success
from swift.common.middleware import acl as swift_acl
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized
from swift.common.utils import register_swift_info
from swift.proxy.controllers.base import get_account_info
import functools
PROJECT_DOMAIN_ID_HEADER = 'x-account-project-domain-id'
PROJECT_DOMAIN_ID_SYSMETA_HEADER = \
get_sys_meta_prefix('account') + 'project-domain-id'
# a string that is unique w.r.t valid ids
UNKNOWN_ID = '_unknown'
class KeystoneAuth(object):
"""Swift middleware to Keystone authorization system.
In Swift's proxy-server.conf add this middleware to your pipeline::
[pipeline:main]
pipeline = catch_errors cache authtoken keystoneauth proxy-server
Make sure you have the authtoken middleware before the
keystoneauth middleware.
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from swift.common import utils as swift_utils
from swift.common.http import is_success
from swift.common.middleware import acl as swift_acl
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized
from swift.common.utils import config_read_reseller_options, list_from_csv
from swift.proxy.controllers.base import get_account_info
import functools
PROJECT_DOMAIN_ID_HEADER = "x-account-project-domain-id"
PROJECT_DOMAIN_ID_SYSMETA_HEADER = get_sys_meta_prefix("account") + "project-domain-id"
# a string that is unique w.r.t valid ids
UNKNOWN_ID = "_unknown"
class KeystoneAuth(object):
"""Swift middleware to Keystone authorization system.
In Swift's proxy-server.conf add this middleware to your pipeline::
[pipeline:main]
pipeline = catch_errors cache authtoken keystoneauth proxy-server
Make sure you have the authtoken middleware before the
keystoneauth middleware.
def purge_crypto_sysmeta_headers(headers):
return [
h for h in headers if not h[0].lower().startswith((
get_object_transient_sysmeta('crypto-'),
get_sys_meta_prefix('object') + 'crypto-'))
]
"""
import requests
import time
from datetime import datetime, timedelta
from swift.common import http, swob, utils, wsgi
from swift.common.request_helpers import get_sys_meta_prefix
from swift.proxy.controllers.base import get_account_info, get_container_info
HOST = "localhost"
PORT = "8081"
DEFAULT_TRASH_PREFIX = ".trash-"
DEFAULT_TRASH_LIFETIME = 86400 * 90 # 90 days expressed in seconds
SYSMETA_UNDELETE_ENABLED = "undelete-enabled"
SYSMETA_ACCOUNT = get_sys_meta_prefix('account') + SYSMETA_UNDELETE_ENABLED
SYSMETA_CONTAINER = get_sys_meta_prefix('container') + SYSMETA_UNDELETE_ENABLED
try:
from swift.common.request_helpers import close_if_possible
except ImportError:
# Pre-1.13.0 (ref. https://github.com/openstack/swift/commit/1f67eb7)
def close_if_possible(maybe_closable):
close_method = getattr(maybe_closable, 'close', None)
if callable(close_method):
return close_method()
def friendly_error(orig_error):
return "Error copying object to trash:\n" + orig_error
check_path_header, get_container_update_override_key, \
update_ignore_range_header
from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \
HTTPException, HTTPConflict, HTTPPreconditionFailed, wsgi_quote, \
wsgi_unquote, status_map, normalize_etag
from swift.common.http import is_success, HTTP_NOT_FOUND
from swift.common.exceptions import LinkIterError
from swift.common.header_key_dict import HeaderKeyDict
DEFAULT_SYMLOOP_MAX = 2
# Header values for symlink target path strings will be quoted values.
TGT_OBJ_SYMLINK_HDR = 'x-symlink-target'
TGT_ACCT_SYMLINK_HDR = 'x-symlink-target-account'
TGT_ETAG_SYMLINK_HDR = 'x-symlink-target-etag'
TGT_BYTES_SYMLINK_HDR = 'x-symlink-target-bytes'
TGT_OBJ_SYSMETA_SYMLINK_HDR = get_sys_meta_prefix('object') + 'symlink-target'
TGT_ACCT_SYSMETA_SYMLINK_HDR = \
get_sys_meta_prefix('object') + 'symlink-target-account'
TGT_ETAG_SYSMETA_SYMLINK_HDR = \
get_sys_meta_prefix('object') + 'symlink-target-etag'
TGT_BYTES_SYSMETA_SYMLINK_HDR = \
get_sys_meta_prefix('object') + 'symlink-target-bytes'
SYMLOOP_EXTEND = get_sys_meta_prefix('object') + 'symloop-extend'
ALLOW_RESERVED_NAMES = get_sys_meta_prefix('object') + 'allow-reserved-names'
def _validate_and_prep_request_headers(req):
"""
Validate that the value from x-symlink-target header is well formatted
and that the x-symlink-target-etag header (if present) does not contain
problematic characters. We assume the caller ensures that
from swift.common.request_helpers import (remove_items, get_sys_meta_prefix,
OBJECT_TRANSIENT_SYSMETA_PREFIX)
from six.moves.urllib.parse import urlsplit
import re
#: A list of python regular expressions that will be used to
#: match against inbound request headers. Matching headers will
#: be removed from the request.
# Exclude headers starting with a sysmeta prefix.
# Exclude headers starting with object transient system metadata prefix.
# Exclude headers starting with an internal backend header prefix.
# If adding to this list, note that these are regex patterns,
# so use a trailing $ to constrain to an exact header match
# rather than prefix match.
inbound_exclusions = [
get_sys_meta_prefix('account'),
get_sys_meta_prefix('container'),
get_sys_meta_prefix('object'), OBJECT_TRANSIENT_SYSMETA_PREFIX, 'x-backend'
]
#: A list of python regular expressions that will be used to
#: match against outbound response headers. Matching headers will
#: be removed from the response.
outbound_exclusions = inbound_exclusions
def make_exclusion_test(exclusions):
expr = '|'.join(exclusions)
test = re.compile(expr, re.IGNORECASE)
return test.match
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
ai = self.account_info(self.account_name, req)
auto_account = self.account_name.startswith(
self.app.auto_create_account_prefix)
if not (auto_account or ai[1]):
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
# Don't cache this. The lack of account will be cached, and that
# is sufficient.
return HTTPNotFound(request=req)
# The read-modify-write of params here is because the Request.params
# getter dynamically generates a dict of params from the query string;
# the setter must be called for new params to update the query string.
params = req.params
params['format'] = 'json'
# x-backend-record-type may be sent via internal client e.g. from
# the sharder or in probe tests
record_type = req.headers.get('X-Backend-Record-Type', '').lower()
if not record_type:
record_type = 'auto'
req.headers['X-Backend-Record-Type'] = 'auto'
params['states'] = 'listing'
req.params = params
memcache = cache_from_env(req.environ, True)
if (req.method == 'GET' and record_type != 'object'
and self.app.recheck_listing_shard_ranges > 0 and memcache
and get_param(req, 'states') == 'listing'
and not config_true_value(
req.headers.get('x-backend-include-deleted', False))):
# This GET might be served from cache or might populate cache.
# 'x-backend-include-deleted' is not usually expected in requests
# to the proxy (it is used from sharder to container servers) but
# it is included in the conditions just in case because we don't
# cache deleted shard ranges.
resp = self._GET_using_cache(req)
else:
resp = self._GETorHEAD_from_backend(req)
resp_record_type = resp.headers.get('X-Backend-Record-Type', '')
if all((req.method == "GET", record_type == 'auto',
resp_record_type.lower() == 'shard')):
resp = self._get_from_shards(req, resp)
if not config_true_value(resp.headers.get('X-Backend-Cached-Results')):
# Cache container metadata. We just made a request to a storage
# node and got up-to-date information for the container.
resp.headers['X-Backend-Recheck-Container-Existence'] = str(
self.app.recheck_container_existence)
set_info_cache(self.app, req.environ, self.account_name,
self.container_name, resp)
if 'swift.authorize' in req.environ:
req.acl = resp.headers.get('x-container-read')
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
if not req.environ.get('swift_owner', False):
for key in self.app.swift_owner_headers:
if key in resp.headers:
del resp.headers[key]
# Expose sharding state in reseller requests
if req.environ.get('reseller_request', False):
resp.headers['X-Container-Sharding'] = config_true_value(
resp.headers.get(
get_sys_meta_prefix('container') + 'Sharding', 'False'))
return resp
* Move to separate account, not container, for trash. This requires Swift to
allow cross-account COPY requests.
* If block_trash_deletes is on, modify the Allow header in responses (both
OPTIONS responses and any other 405 response).
"""
from swift.common import http, swob, utils, wsgi
from swift.common.request_helpers import get_sys_meta_prefix
from swift.proxy.controllers.base import get_account_info, get_container_info
DEFAULT_TRASH_PREFIX = ".trash-"
DEFAULT_TRASH_LIFETIME = 86400 * 90 # 90 days expressed in seconds
SYSMETA_UNDELETE_ENABLED = "undelete-enabled"
SYSMETA_ACCOUNT = get_sys_meta_prefix('account') + SYSMETA_UNDELETE_ENABLED
SYSMETA_CONTAINER = get_sys_meta_prefix('container') + SYSMETA_UNDELETE_ENABLED
try:
from swift.common.request_helpers import close_if_possible
except ImportError:
# Pre-1.13.0 (ref. https://github.com/openstack/swift/commit/1f67eb7)
def close_if_possible(maybe_closable):
close_method = getattr(maybe_closable, 'close', None)
if callable(close_method):
return close_method()
def friendly_error(orig_error):
return "Error copying object to trash:\n" + orig_error
def sysmeta_to_client(self, resp, req_type):
prefix = get_sys_meta_prefix(req_type) + 'default-'
for header, value in resp.headers.items():
if header.lower().startswith(prefix):
client_header = 'x-default-%s' % header[len(prefix):]
resp.headers[client_header] = value
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from builtins import object
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.swob import Request
SYSMETA_HEADER = get_sys_meta_prefix('account') + 'project-domain-id'
OVERRIDE_HEADER = 'x-account-project-domain-id-override'
class DomainOverrideMiddleware(object):
def __init__(self, app, **kwargs):
self.app = app
@classmethod
def factory(cls, global_config, **local_config):
def _factory(app):
return cls(app, **local_config)
return _factory
def __call__(self, environ, start_response):
# only reseller may invoke the override
"""
from swift.common.swob import wsgify
from swift.common.utils import get_logger
from swift.common.request_helpers import remove_items, get_sys_meta_prefix
import re
#: A list of python regular expressions that will be used to
#: match against inbound request headers. Matching headers will
#: be removed from the request.
# Exclude headers starting with a sysmeta prefix.
# If adding to this list, note that these are regex patterns,
# so use a trailing $ to constrain to an exact header match
# rather than prefix match.
inbound_exclusions = [
get_sys_meta_prefix('account'),
get_sys_meta_prefix('container'),
get_sys_meta_prefix('object')
]
# 'x-object-sysmeta' is reserved in anticipation of future support
# for system metadata being applied to objects
#: A list of python regular expressions that will be used to
#: match against outbound response headers. Matching headers will
#: be removed from the response.
outbound_exclusions = inbound_exclusions
def make_exclusion_test(exclusions):
expr = '|'.join(exclusions)
test = re.compile(expr, re.IGNORECASE)
from swift.common.utils import register_swift_info
from swift.common.utils import split_path
from swift.proxy.controllers.base import get_container_info
from eventlet.green import urllib2
from eventlet import Timeout
import base64
import json
import ssl
import urlparse
# Container system metadata
# x-container-sysmeta-webhookk & x-container-sysmeta-webhook-auth
# Add x-container-sysmeta-webhook-get for hooks called during get requests
SYSMETA_WEBHOOK = get_sys_meta_prefix('container') + 'webhook'
SYSMETA_WEBHOOK_AUTH = get_sys_meta_prefix('container') + 'webhook-auth'
SYSMETA_WEBHOOK_GET = get_sys_meta_prefix('container') + 'webhook-get'
SYSMETA_WEBHOOK_GET_AUTH = get_sys_meta_prefix(
'container') + 'webhook-get-auth'
class WebhookMiddleware(object):
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route='webhook')
@wsgify
def __call__(self, req):
obj = None
container = None
MD5_OF_EMPTY_STRING, closing_if_possible
from swift.common.constraints import check_account_format
from swift.common.wsgi import WSGIContext, make_subrequest
from swift.common.request_helpers import get_sys_meta_prefix, \
check_path_header
from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \
HTTPException, HTTPConflict, HTTPPreconditionFailed
from swift.common.http import is_success
from swift.common.exceptions import LinkIterError
from swift.common.header_key_dict import HeaderKeyDict
DEFAULT_SYMLOOP_MAX = 2
# Header values for symlink target path strings will be quoted values.
TGT_OBJ_SYMLINK_HDR = 'x-symlink-target'
TGT_ACCT_SYMLINK_HDR = 'x-symlink-target-account'
TGT_OBJ_SYSMETA_SYMLINK_HDR = get_sys_meta_prefix('object') + 'symlink-target'
TGT_ACCT_SYSMETA_SYMLINK_HDR = \
get_sys_meta_prefix('object') + 'symlink-target-account'
def _check_symlink_header(req):
"""
Validate that the value from x-symlink-target header is
well formatted. We assume the caller ensures that
x-symlink-target header is present in req.headers.
:param req: HTTP request object
:raise: HTTPPreconditionFailed if x-symlink-target value
is not well formatted.
:raise: HTTPBadRequest if the x-symlink-target value points to the request
path.
def purge_crypto_sysmeta_headers(headers):
return [h for h in headers if not
h[0].lower().startswith(
(get_object_transient_sysmeta('crypto-'),
get_sys_meta_prefix('object') + 'crypto-'))]
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from swift.common import utils as swift_utils
from swift.common.http import is_success
from swift.common.middleware import acl as swift_acl
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized
from swift.common.utils import config_read_reseller_options, list_from_csv
from swift.proxy.controllers.base import get_account_info
import functools
PROJECT_DOMAIN_ID_HEADER = 'x-account-project-domain-id'
PROJECT_DOMAIN_ID_SYSMETA_HEADER = \
get_sys_meta_prefix('account') + 'project-domain-id'
# a string that is unique w.r.t valid ids
UNKNOWN_ID = '_unknown'
class KeystoneAuth(object):
"""Swift middleware to Keystone authorization system.
In Swift's proxy-server.conf add this keystoneauth middleware and the
authtoken middleware to your pipeline. Make sure you have the authtoken
middleware before the keystoneauth middleware.
The authtoken middleware will take care of validating the user and
keystoneauth will authorize access.
The sample proxy-server.conf shows a sample pipeline that uses keystone.
URL = "http://141.115.103.32:8081"
ENDPOINT_PATH = "/api/experimental"
DAG_NAME = "test"
#
# import ConfigParser
# config = ConfigParser.ConfigParser().read(CONFIG_PATH).sections()
import six
if six.PY3:
from eventlet.green.urllib import request as urllib2
else:
from eventlet.green import urllib2
import requests
# x-container-sysmeta-webhook
SYSMETA_WEBHOOK = get_sys_meta_prefix('container') + 'webhook'
import json
class NewDataTriggerMiddleware(object):
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route='newdatatrigger')
@wsgify
def __call__(self, req):
print(req)
print(req.headers)
print(req.path_info)
obj = None
try:
