def POST(self, req): """HTTP POST request handler.""" error_response = \ self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response if not req.environ.get('swift_owner'): for key in self.app.swift_owner_headers: req.headers.pop(key, None) if req.environ.get('reseller_request', False) and \ 'X-Container-Sharding' in req.headers: req.headers[get_sys_meta_prefix('container') + 'Sharding'] = \ str(config_true_value(req.headers['X-Container-Sharding'])) account_partition, accounts, container_count = \ self.account_info(self.account_name, req) if not accounts: return HTTPNotFound(request=req) container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = self.generate_request_headers(req, transfer=True) clear_info_cache(self.app, req.environ, self.account_name, self.container_name) resp = self.make_requests( req, self.app.container_ring, container_partition, 'POST', req.swift_entity_path, [headers] * len(containers)) return resp
def extract_acl_and_report_errors(self, req): """ Return a user-readable string indicating the errors in the input ACL, or None if there are no errors. """ acl_header = 'x-account-access-control' acl_data = req.headers.get(acl_header) result = parse_acl(version=2, data=acl_data) if result is None: return 'Syntax error in input (%r)' % acl_data tempauth_acl_keys = 'admin read-write read-only'.split() for key in result: # While it is possible to construct auth systems that collaborate # on ACLs, TempAuth is not such an auth system. At this point, # it thinks it is authoritative. if key not in tempauth_acl_keys: return 'Key %r not recognized' % key for key in tempauth_acl_keys: if key not in result: continue if not isinstance(result[key], list): return 'Value for key %r must be a list' % key for grantee in result[key]: if not isinstance(grantee, str): return 'Elements of %r list must be strings' % key # Everything looks fine, no errors found internal_hdr = get_sys_meta_prefix('account') + 'core-access-control' req.headers[internal_hdr] = req.headers.pop(acl_header) return None
def client_to_sysmeta(self, req, req_type): subresources = { 'account': ('container', 'object'), 'container': ('object', ), }.get(req_type, ()) header_formats = ( ('x-remove-default-%s-', True), ('x-default-%s-', False), ) for header_format, clear in header_formats: for header, value in req.headers.items(): for subresource in subresources: prefix = header_format % subresource if header.lower().startswith(prefix): header_to_default = header[len(prefix):].lower() if header_to_default.startswith(BLACKLIST_PREFIXES): continue if header_to_default in BLACKLIST: continue sysmeta_header = '%sdefault-%s-%s' % ( get_sys_meta_prefix(req_type), subresource, header_to_default) req.headers[sysmeta_header] = '' if clear else value
def test_headers_to_object_info_sys_meta(self): prefix = get_sys_meta_prefix("object") headers = {"%sWhatevs" % prefix: 14, "%ssomethingelse" % prefix: 0} resp = headers_to_object_info(headers.items(), 200) self.assertEquals(len(resp["sysmeta"]), 2) self.assertEquals(resp["sysmeta"]["whatevs"], 14) self.assertEquals(resp["sysmeta"]["somethingelse"], 0)
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" ai = self.account_info(self.account_name, req) auto_account = self.account_name.startswith( self.app.auto_create_account_prefix) if not (auto_account or ai[1]): if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: # Don't cache this. It doesn't reflect the state of the # container, just that the user can't access it. return aresp # Don't cache this. The lack of account will be cached, and that # is sufficient. return HTTPNotFound(request=req) part = self.app.container_ring.get_part( self.account_name, self.container_name) concurrency = self.app.container_ring.replica_count \ if self.app.concurrent_gets else 1 node_iter = self.app.iter_nodes(self.app.container_ring, part) params = req.params params['format'] = 'json' record_type = req.headers.get('X-Backend-Record-Type', '').lower() if not record_type: record_type = 'auto' req.headers['X-Backend-Record-Type'] = 'auto' params['states'] = 'listing' req.params = params resp = self.GETorHEAD_base( req, _('Container'), node_iter, part, req.swift_entity_path, concurrency) resp_record_type = resp.headers.get('X-Backend-Record-Type', '') if all((req.method == "GET", record_type == 'auto', resp_record_type.lower() == 'shard')): resp = self._get_from_shards(req, resp) # Cache this. We just made a request to a storage node and got # up-to-date information for the container. resp.headers['X-Backend-Recheck-Container-Existence'] = str( self.app.recheck_container_existence) set_info_cache(self.app, req.environ, self.account_name, self.container_name, resp) if 'swift.authorize' in req.environ: req.acl = resp.headers.get('x-container-read') aresp = req.environ['swift.authorize'](req) if aresp: # Don't cache this. It doesn't reflect the state of the # container, just that the user can't access it. return aresp if not req.environ.get('swift_owner', False): for key in self.app.swift_owner_headers: if key in resp.headers: del resp.headers[key] # Expose sharding state in reseller requests if req.environ.get('reseller_request', False): resp.headers['X-Container-Sharding'] = config_true_value( resp.headers.get(get_sys_meta_prefix('container') + 'Sharding', 'False')) return resp
def test_headers_to_object_info_sys_meta(self): prefix = get_sys_meta_prefix('object') headers = {'%sWhatevs' % prefix: 14, '%ssomethingelse' % prefix: 0} resp = headers_to_object_info(headers.items(), 200) self.assertEqual(len(resp['sysmeta']), 2) self.assertEqual(resp['sysmeta']['whatevs'], 14) self.assertEqual(resp['sysmeta']['somethingelse'], 0)
def add_acls_from_sys_metadata(self, resp): if resp.environ["REQUEST_METHOD"] in ("HEAD", "GET", "PUT", "POST"): prefix = get_sys_meta_prefix("account") + "core-" name = "access-control" (extname, intname) = ("x-account-" + name, prefix + name) acl_dict = parse_acl(version=2, data=resp.headers.pop(intname)) if acl_dict: # treat empty dict as empty header resp.headers[extname] = format_acl(version=2, acl_dict=acl_dict)
def add_acls_from_sys_metadata(self, resp): if resp.environ['REQUEST_METHOD'] in ('HEAD', 'GET', 'PUT', 'POST'): prefix = get_sys_meta_prefix('account') + 'core-' name = 'access-control' (extname, intname) = ('x-account-' + name, prefix + name) acl_dict = parse_acl(version=2, data=resp.headers.pop(intname)) if acl_dict: # treat empty dict as empty header resp.headers[extname] = format_acl( version=2, acl_dict=acl_dict)
def _make_user_and_sys_acl_headers_data(self): acl = { 'admin': ['AUTH_alice', 'AUTH_bob'], 'read-write': ['AUTH_carol'], 'read-only': [], } user_prefix = 'x-account-' # external, user-facing user_headers = {(user_prefix + 'access-control'): format_acl( version=2, acl_dict=acl)} sys_prefix = get_sys_meta_prefix('account') # internal, system-facing sys_headers = {(sys_prefix + 'core-access-control'): format_acl( version=2, acl_dict=acl)} return user_headers, sys_headers
def PUT(self, req): """HTTP PUT request handler.""" error_response = \ self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response policy_index = self._convert_policy_to_index(req) if not req.environ.get('swift_owner'): for key in self.app.swift_owner_headers: req.headers.pop(key, None) if req.environ.get('reseller_request', False) and \ 'X-Container-Sharding' in req.headers: req.headers[get_sys_meta_prefix('container') + 'Sharding'] = \ str(config_true_value(req.headers['X-Container-Sharding'])) length_limit = self.get_name_length_limit() if len(self.container_name) > length_limit: body = 'Container name length of %d longer than %d' % ( len(self.container_name), length_limit) resp = HTTPBadRequest(request=req, body=body) return resp account_partition, accounts, container_count = \ self.account_info(self.account_name, req) if not accounts and self.app.account_autocreate: if not self.autocreate_account(req, self.account_name): return HTTPServiceUnavailable(request=req) account_partition, accounts, container_count = \ self.account_info(self.account_name, req) if not accounts: return HTTPNotFound(request=req) if 0 < self.app.max_containers_per_account <= container_count and \ self.account_name not in self.app.max_containers_whitelist: container_info = \ self.container_info(self.account_name, self.container_name, req) if not is_success(container_info.get('status')): body = 'Reached container limit of %s' % ( self.app.max_containers_per_account, ) resp = HTTPForbidden(request=req, body=body) return resp container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = self._backend_requests(req, len(containers), account_partition, accounts, policy_index) resp = self.make_requests( req, self.app.container_ring, container_partition, 'PUT', req.swift_entity_path, headers) clear_info_cache(self.app, req.environ, self.account_name, self.container_name) return resp
def test_sys_meta_headers_PUT(self): sys_meta_key = '%stest' % get_sys_meta_prefix('container') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Container-Meta-Test' hdrs_in = {sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0'} req = Request.blank('/v1/a/c', headers=hdrs_in, method='PUT') self.storage.container.container_create = Mock() req.get_response(self.app) meta = \ self.storage.container.container_create.call_args[1]['properties'] self.assertEqual(meta[sys_meta_key], 'foo') self.assertEqual(meta[user_meta_key], 'bar')
def test_sys_meta_headers_POST(self): # check that headers in sys meta namespace make it through # the proxy controller sys_meta_key = '%stest' % get_sys_meta_prefix('account') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Account-Meta-Test' hdrs_in = {sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0'} req = Request.blank('/v1/a', headers=hdrs_in, method='POST') self.storage.account_set_properties = Mock() req.get_response(self.app) meta = self.storage.account_set_properties.call_args[1]['properties'] self.assertEqual(meta[sys_meta_key], 'foo') self.assertEqual(meta[user_meta_key], 'bar')
def test_sys_meta_headers_POST(self): # check that headers in sys meta namespace make it through # the container controller sys_meta_key = '%stest' % get_sys_meta_prefix('container') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Container-Meta-Test' hdrs_in = {sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0'} req = Request.blank('/v1/a/c', headers=hdrs_in, method='POST') self.storage.container.container_set_properties = Mock( return_value="") req.get_response(self.app) meta = self.storage.container.container_set_properties.call_args[0][2] self.assertEqual(meta[sys_meta_key], 'foo') self.assertEqual(meta[user_meta_key], 'bar')
def handle_container_request(self, env, start_response): app_resp = self._app_call(env) if self._response_headers is None: self._response_headers = [] sysmeta_version_hdr = get_sys_meta_prefix('container') + \ 'versions-location' location = '' for key, val in self._response_headers: if key.lower() == sysmeta_version_hdr: location = val if location: self._response_headers.extend([('X-Versions-Location', location)]) start_response(self._response_status, self._response_headers, self._response_exc_info) return app_resp
def container_request(self, req, start_response, enabled): sysmeta_version_hdr = get_sys_meta_prefix('container') + \ 'versions-location' # set version location header as sysmeta if 'X-Versions-Location' in req.headers: val = req.headers.get('X-Versions-Location') if val: # differently from previous version, we are actually # returning an error if user tries to set versions location # while feature is explicitly disabled. if not config_true_value(enabled) and \ req.method in ('PUT', 'POST'): raise HTTPPreconditionFailed( request=req, content_type='text/plain', body='Versioned Writes is disabled') location = check_container_format(req, val) req.headers[sysmeta_version_hdr] = location # reset original header to maintain sanity # now only sysmeta is source of Versions Location req.headers['X-Versions-Location'] = '' # if both headers are in the same request # adding location takes precedence over removing if 'X-Remove-Versions-Location' in req.headers: del req.headers['X-Remove-Versions-Location'] else: # empty value is the same as X-Remove-Versions-Location req.headers['X-Remove-Versions-Location'] = 'x' # handle removing versions container val = req.headers.get('X-Remove-Versions-Location') if val: req.headers.update({sysmeta_version_hdr: ''}) req.headers.update({'X-Versions-Location': ''}) del req.headers['X-Remove-Versions-Location'] # send request and translate sysmeta headers from response vw_ctx = VersionedWritesContext(self.app, self.logger) return vw_ctx.handle_container_request(req.environ, start_response)
def test_sys_meta_headers_PUT(self): # check that headers in sys meta namespace make it through # the proxy controller sys_meta_key = '%stest' % get_sys_meta_prefix('account') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Account-Meta-Test' # allow PUTs to account... self.app.allow_account_management = True hdrs_in = { sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0' } req = Request.blank('/v1/a', headers=hdrs_in, method='PUT') self.storage.account_create = Mock() self.storage.account_set_properties = Mock() req.get_response(self.app) meta = self.storage.account_set_properties.call_args[0][1] self.assertEqual(meta[sys_meta_key], 'foo') self.assertEqual(meta[user_meta_key], 'bar')
def test_sys_meta_headers_POST(self): # check that headers in sys meta namespace make it through # the container controller sys_meta_key = "%stest" % get_sys_meta_prefix("container") sys_meta_key = sys_meta_key.title() user_meta_key = "X-Container-Meta-Test" controller = proxy_server.ContainerController(self.app, "a", "c") context = {} callback = self._make_callback_func(context) hdrs_in = {sys_meta_key: "foo", user_meta_key: "bar", "x-timestamp": "1.0"} req = Request.blank("/v1/a/c", headers=hdrs_in) with mock.patch( "swift.proxy.controllers.base.http_connect", fake_http_connect(200, 200, give_connect=callback) ): controller.POST(req) self.assertEqual(context["method"], "POST") self.assertTrue(sys_meta_key in context["headers"]) self.assertEqual(context["headers"][sys_meta_key], "foo") self.assertTrue(user_meta_key in context["headers"]) self.assertEqual(context["headers"][user_meta_key], "bar") self.assertNotEqual(context["headers"]["x-timestamp"], "1.0")
def test_sys_meta_headers_POST(self): # check that headers in sys meta namespace make it through # the proxy controller sys_meta_key = '%stest' % get_sys_meta_prefix('account') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Account-Meta-Test' controller = proxy_server.AccountController(self.app, 'a') context = {} callback = self._make_callback_func(context) hdrs_in = {sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0'} req = Request.blank('/v1/a', headers=hdrs_in) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, give_connect=callback)): controller.POST(req) self.assertEqual(context['method'], 'POST') self.assertTrue(sys_meta_key in context['headers']) self.assertEqual(context['headers'][sys_meta_key], 'foo') self.assertTrue(user_meta_key in context['headers']) self.assertEqual(context['headers'][user_meta_key], 'bar') self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
""" from swift.common.swob import Request from swift.common.utils import get_logger from swift.common.request_helpers import remove_items, get_sys_meta_prefix import re #: A list of python regular expressions that will be used to #: match against inbound request headers. Matching headers will #: be removed from the request. # Exclude headers starting with a sysmeta prefix. # If adding to this list, note that these are regex patterns, # so use a trailing $ to constrain to an exact header match # rather than prefix match. inbound_exclusions = [get_sys_meta_prefix('account'), get_sys_meta_prefix('container'), get_sys_meta_prefix('object'), 'x-backend'] # 'x-object-sysmeta' is reserved in anticipation of future support # for system metadata being applied to objects #: A list of python regular expressions that will be used to #: match against outbound response headers. Matching headers will #: be removed from the response. outbound_exclusions = inbound_exclusions def make_exclusion_test(exclusions): expr = '|'.join(exclusions)
def _save_bucket_name(self, env): req = Request(env) account, container, obj = self._extract_path(req.path_info) sys_meta_key = '%soio-bucket-name' % get_sys_meta_prefix('object') req.headers[sys_meta_key] = container
swift.defaulter_hook This is a callback that may be used to populate defaults for subrequests. It will only modify PUT requests. It accepts a swob.Request as an argument. """ from swift.common.request_helpers import get_sys_meta_prefix from swift.common.swob import wsgify from swift.common.utils import config_true_value from swift.common.utils import register_swift_info from swift.proxy.controllers.base import get_account_info from swift.proxy.controllers.base import get_container_info BLACKLIST = set('x-timestamp') BLACKLIST_PREFIXES = ( get_sys_meta_prefix('account'), get_sys_meta_prefix('container'), get_sys_meta_prefix('object'), 'x-backend-', ) CALLBACK_ENV_KEY = 'swift.defaulter_hook' HEADERS_ENV_KEY = 'swift.defaulter_headers' class DefaulterMiddleware(object): def __init__(self, app, config): self.app = app self.conf = config @wsgify def __call__(self, req):
FileLikeIter, ) from swift.common.request_helpers import get_sys_meta_prefix, copy_header_subset from swift.common.wsgi import WSGIContext, make_pre_authed_request from swift.common.swob import Request, HTTPException, HTTPRequestEntityTooLarge from swift.common.constraints import check_container_format, MAX_FILE_SIZE from swift.proxy.controllers.base import get_container_info from swift.common.http import is_success, is_client_error, HTTP_NOT_FOUND from swift.common.swob import HTTPPreconditionFailed, HTTPServiceUnavailable, HTTPServerError, HTTPBadRequest from swift.common.exceptions import ListingIterNotFound, ListingIterError VERSIONING_MODES = ("stack", "history") DELETE_MARKER_CONTENT_TYPE = "application/x-deleted;swift_versions_deleted=1" VERSIONS_LOC_CLIENT = "x-versions-location" VERSIONS_LOC_SYSMETA = get_sys_meta_prefix("container") + "versions-location" VERSIONS_MODE_CLIENT = "x-versions-mode" VERSIONS_MODE_SYSMETA = get_sys_meta_prefix("container") + "versions-mode" class VersionedWritesContext(WSGIContext): def __init__(self, wsgi_app, logger): WSGIContext.__init__(self, wsgi_app) self.logger = logger def _listing_iter(self, account_name, lcontainer, lprefix, req): try: for page in self._listing_pages_iter(account_name, lcontainer, lprefix, req.environ): for item in page: yield item except ListingIterNotFound:
MD5_OF_EMPTY_STRING, closing_if_possible, quote from swift.common.constraints import check_account_format from swift.common.wsgi import WSGIContext, make_subrequest from swift.common.request_helpers import get_sys_meta_prefix, \ check_path_header from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \ HTTPException, HTTPConflict, HTTPPreconditionFailed from swift.common.http import is_success from swift.common.exceptions import LinkIterError from swift.common.header_key_dict import HeaderKeyDict DEFAULT_SYMLOOP_MAX = 2 # Header values for symlink target path strings will be quoted values. TGT_OBJ_SYMLINK_HDR = 'x-symlink-target' TGT_ACCT_SYMLINK_HDR = 'x-symlink-target-account' TGT_OBJ_SYSMETA_SYMLINK_HDR = get_sys_meta_prefix('object') + 'symlink-target' TGT_ACCT_SYSMETA_SYMLINK_HDR = \ get_sys_meta_prefix('object') + 'symlink-target-account' def _check_symlink_header(req): """ Validate that the value from x-symlink-target header is well formatted. We assume the caller ensures that x-symlink-target header is present in req.headers. :param req: HTTP request object :raise: HTTPPreconditionFailed if x-symlink-target value is not well formatted. :raise: HTTPBadRequest if the x-symlink-target value points to the request path.
get_sys_meta_prefix, update_etag_is_at_header from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success from swift.common.wsgi import WSGIContext, make_subrequest from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk DEFAULT_RATE_LIMIT_UNDER_SIZE = 1024 * 1024 # 1 MiB DEFAULT_MAX_MANIFEST_SEGMENTS = 1000 DEFAULT_MAX_MANIFEST_SIZE = 1024 * 1024 * 2 # 2 MiB REQUIRED_SLO_KEYS = set(['path', 'etag', 'size_bytes']) OPTIONAL_SLO_KEYS = set(['range']) ALLOWED_SLO_KEYS = REQUIRED_SLO_KEYS | OPTIONAL_SLO_KEYS SYSMETA_SLO_ETAG = get_sys_meta_prefix('object') + 'slo-etag' SYSMETA_SLO_SIZE = get_sys_meta_prefix('object') + 'slo-size' def parse_and_validate_input(req_body, req_path): """ Given a request body, parses it and returns a list of dictionaries. The output structure is nearly the same as the input structure, but it is not an exact copy. Given a valid input dictionary `d_in`, its corresponding output dictionary `d_out` will be as follows: * d_out['etag'] == d_in['etag'] * d_out['path'] == d_in['path']
HOP_BY_HOP_HEADERS = set([ 'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailer', 'transfer-encoding', 'upgrade', ]) PROPAGATED_HDRS = ['x-container-read', 'x-container-write'] MIGRATOR_HEADER = 'multi-cloud-internal-migrator' SHUNT_BYPASS_HEADER = 'x-cloud-sync-shunt-bypass' ACCOUNT_ACL_KEY = 'x-account-access-control' SYSMETA_ACCOUNT_ACL_KEY = \ get_sys_meta_prefix('account') + 'core-access-control' class MigrationContainerStates(object): '''Possible states of the migrated containers. When a container is migrated, we have to handle the possible state transitions it could go through. Initially, a container only exists in the source cluster. Once we create it in the destination cluster, we tag it to be in the MIGRATING state. In this state, we process any new objects or updated metadata. When an object is removed, we will also remove it from the destination container. When a container is removed on the source and the destination container is in the MIGRATING state, it will be removed.
from swift.common.request_helpers import get_sys_meta_prefix, \ copy_header_subset from swift.common.wsgi import WSGIContext, make_pre_authed_request from swift.common.swob import (Request, HTTPException, HTTPRequestEntityTooLarge) from swift.common.constraints import check_container_format, MAX_FILE_SIZE from swift.proxy.controllers.base import get_container_info from swift.common.http import (is_success, is_client_error, HTTP_NOT_FOUND) from swift.common.swob import HTTPPreconditionFailed, HTTPServiceUnavailable, \ HTTPServerError, HTTPBadRequest from swift.common.exceptions import (ListingIterNotFound, ListingIterError) DELETE_MARKER_CONTENT_TYPE = 'application/x-deleted;swift_versions_deleted=1' CLIENT_VERSIONS_LOC = 'x-versions-location' CLIENT_HISTORY_LOC = 'x-history-location' SYSMETA_VERSIONS_LOC = get_sys_meta_prefix('container') + 'versions-location' SYSMETA_VERSIONS_MODE = get_sys_meta_prefix('container') + 'versions-mode' class VersionedWritesContext(WSGIContext): def __init__(self, wsgi_app, logger): WSGIContext.__init__(self, wsgi_app) self.logger = logger def _listing_iter(self, account_name, lcontainer, lprefix, req): try: for page in self._listing_pages_iter(account_name, lcontainer, lprefix, req.environ): for item in page: yield item except ListingIterNotFound:
from swift.common.swob import ( Request, HTTPException, HTTPRequestEntityTooLarge) from swift.common.constraints import check_container_format, MAX_FILE_SIZE from swift.proxy.controllers.base import get_container_info from swift.common.http import ( is_success, is_client_error, HTTP_NOT_FOUND) from swift.common.swob import HTTPPreconditionFailed, HTTPServiceUnavailable, \ HTTPServerError, HTTPBadRequest from swift.common.exceptions import ( ListingIterNotFound, ListingIterError) DELETE_MARKER_CONTENT_TYPE = 'application/x-deleted;swift_versions_deleted=1' CLIENT_VERSIONS_LOC = 'x-versions-location' CLIENT_HISTORY_LOC = 'x-history-location' SYSMETA_VERSIONS_LOC = get_sys_meta_prefix('container') + 'versions-location' SYSMETA_VERSIONS_MODE = get_sys_meta_prefix('container') + 'versions-mode' class VersionedWritesContext(WSGIContext): def __init__(self, wsgi_app, logger): WSGIContext.__init__(self, wsgi_app) self.logger = logger def _listing_iter(self, account_name, lcontainer, lprefix, req): try: for page in self._listing_pages_iter(account_name, lcontainer, lprefix, req.environ): for item in page: yield item
def get_sys_migrator_header(path_type): if path_type == 'object': return get_object_transient_sysmeta(MIGRATOR_HEADER) return '%s%s' % (get_sys_meta_prefix(path_type), MIGRATOR_HEADER)
def sysmeta_to_client(self, resp, req_type): prefix = get_sys_meta_prefix(req_type) + 'default-' for header, value in resp.headers.items(): if header.lower().startswith(prefix): client_header = 'x-default-%s' % header[len(prefix):] resp.headers[client_header] = value
from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success from swift.common.wsgi import WSGIContext, make_subrequest from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk DEFAULT_RATE_LIMIT_UNDER_SIZE = 1024 * 1024 # 1 MiB DEFAULT_MAX_MANIFEST_SEGMENTS = 1000 DEFAULT_MAX_MANIFEST_SIZE = 1024 * 1024 * 2 # 2 MiB REQUIRED_SLO_KEYS = set(['path']) OPTIONAL_SLO_KEYS = set(['range', 'etag', 'size_bytes']) ALLOWED_SLO_KEYS = REQUIRED_SLO_KEYS | OPTIONAL_SLO_KEYS SYSMETA_SLO_ETAG = get_sys_meta_prefix('object') + 'slo-etag' SYSMETA_SLO_SIZE = get_sys_meta_prefix('object') + 'slo-size' def parse_and_validate_input(req_body, req_path): """ Given a request body, parses it and returns a list of dictionaries. The output structure is nearly the same as the input structure, but it is not an exact copy. Given a valid input dictionary `d_in`, its corresponding output dictionary `d_out` will be as follows: * d_out['etag'] == d_in['etag'] * d_out['path'] == d_in['path']
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from swift.common import utils as swift_utils from swift.common.http import is_success from swift.common.middleware import acl as swift_acl from swift.common.request_helpers import get_sys_meta_prefix from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized from swift.common.utils import register_swift_info from swift.proxy.controllers.base import get_account_info import functools PROJECT_DOMAIN_ID_HEADER = 'x-account-project-domain-id' PROJECT_DOMAIN_ID_SYSMETA_HEADER = \ get_sys_meta_prefix('account') + 'project-domain-id' # a string that is unique w.r.t valid ids UNKNOWN_ID = '_unknown' class KeystoneAuth(object): """Swift middleware to Keystone authorization system. In Swift's proxy-server.conf add this middleware to your pipeline:: [pipeline:main] pipeline = catch_errors cache authtoken keystoneauth proxy-server Make sure you have the authtoken middleware before the keystoneauth middleware.
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from swift.common import utils as swift_utils from swift.common.http import is_success from swift.common.middleware import acl as swift_acl from swift.common.request_helpers import get_sys_meta_prefix from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized from swift.common.utils import config_read_reseller_options, list_from_csv from swift.proxy.controllers.base import get_account_info import functools PROJECT_DOMAIN_ID_HEADER = "x-account-project-domain-id" PROJECT_DOMAIN_ID_SYSMETA_HEADER = get_sys_meta_prefix("account") + "project-domain-id" # a string that is unique w.r.t valid ids UNKNOWN_ID = "_unknown" class KeystoneAuth(object): """Swift middleware to Keystone authorization system. In Swift's proxy-server.conf add this middleware to your pipeline:: [pipeline:main] pipeline = catch_errors cache authtoken keystoneauth proxy-server Make sure you have the authtoken middleware before the keystoneauth middleware.
def purge_crypto_sysmeta_headers(headers): return [ h for h in headers if not h[0].lower().startswith(( get_object_transient_sysmeta('crypto-'), get_sys_meta_prefix('object') + 'crypto-')) ]
""" import requests import time from datetime import datetime, timedelta from swift.common import http, swob, utils, wsgi from swift.common.request_helpers import get_sys_meta_prefix from swift.proxy.controllers.base import get_account_info, get_container_info HOST = "localhost" PORT = "8081" DEFAULT_TRASH_PREFIX = ".trash-" DEFAULT_TRASH_LIFETIME = 86400 * 90 # 90 days expressed in seconds SYSMETA_UNDELETE_ENABLED = "undelete-enabled" SYSMETA_ACCOUNT = get_sys_meta_prefix('account') + SYSMETA_UNDELETE_ENABLED SYSMETA_CONTAINER = get_sys_meta_prefix('container') + SYSMETA_UNDELETE_ENABLED try: from swift.common.request_helpers import close_if_possible except ImportError: # Pre-1.13.0 (ref. https://github.com/openstack/swift/commit/1f67eb7) def close_if_possible(maybe_closable): close_method = getattr(maybe_closable, 'close', None) if callable(close_method): return close_method() def friendly_error(orig_error): return "Error copying object to trash:\n" + orig_error
check_path_header, get_container_update_override_key, \ update_ignore_range_header from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \ HTTPException, HTTPConflict, HTTPPreconditionFailed, wsgi_quote, \ wsgi_unquote, status_map, normalize_etag from swift.common.http import is_success, HTTP_NOT_FOUND from swift.common.exceptions import LinkIterError from swift.common.header_key_dict import HeaderKeyDict DEFAULT_SYMLOOP_MAX = 2 # Header values for symlink target path strings will be quoted values. TGT_OBJ_SYMLINK_HDR = 'x-symlink-target' TGT_ACCT_SYMLINK_HDR = 'x-symlink-target-account' TGT_ETAG_SYMLINK_HDR = 'x-symlink-target-etag' TGT_BYTES_SYMLINK_HDR = 'x-symlink-target-bytes' TGT_OBJ_SYSMETA_SYMLINK_HDR = get_sys_meta_prefix('object') + 'symlink-target' TGT_ACCT_SYSMETA_SYMLINK_HDR = \ get_sys_meta_prefix('object') + 'symlink-target-account' TGT_ETAG_SYSMETA_SYMLINK_HDR = \ get_sys_meta_prefix('object') + 'symlink-target-etag' TGT_BYTES_SYSMETA_SYMLINK_HDR = \ get_sys_meta_prefix('object') + 'symlink-target-bytes' SYMLOOP_EXTEND = get_sys_meta_prefix('object') + 'symloop-extend' ALLOW_RESERVED_NAMES = get_sys_meta_prefix('object') + 'allow-reserved-names' def _validate_and_prep_request_headers(req): """ Validate that the value from x-symlink-target header is well formatted and that the x-symlink-target-etag header (if present) does not contain problematic characters. We assume the caller ensures that
from swift.common.request_helpers import (remove_items, get_sys_meta_prefix, OBJECT_TRANSIENT_SYSMETA_PREFIX) from six.moves.urllib.parse import urlsplit import re #: A list of python regular expressions that will be used to #: match against inbound request headers. Matching headers will #: be removed from the request. # Exclude headers starting with a sysmeta prefix. # Exclude headers starting with object transient system metadata prefix. # Exclude headers starting with an internal backend header prefix. # If adding to this list, note that these are regex patterns, # so use a trailing $ to constrain to an exact header match # rather than prefix match. inbound_exclusions = [ get_sys_meta_prefix('account'), get_sys_meta_prefix('container'), get_sys_meta_prefix('object'), OBJECT_TRANSIENT_SYSMETA_PREFIX, 'x-backend' ] #: A list of python regular expressions that will be used to #: match against outbound response headers. Matching headers will #: be removed from the response. outbound_exclusions = inbound_exclusions def make_exclusion_test(exclusions): expr = '|'.join(exclusions) test = re.compile(expr, re.IGNORECASE) return test.match
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" ai = self.account_info(self.account_name, req) auto_account = self.account_name.startswith( self.app.auto_create_account_prefix) if not (auto_account or ai[1]): if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: # Don't cache this. It doesn't reflect the state of the # container, just that the user can't access it. return aresp # Don't cache this. The lack of account will be cached, and that # is sufficient. return HTTPNotFound(request=req) # The read-modify-write of params here is because the Request.params # getter dynamically generates a dict of params from the query string; # the setter must be called for new params to update the query string. params = req.params params['format'] = 'json' # x-backend-record-type may be sent via internal client e.g. from # the sharder or in probe tests record_type = req.headers.get('X-Backend-Record-Type', '').lower() if not record_type: record_type = 'auto' req.headers['X-Backend-Record-Type'] = 'auto' params['states'] = 'listing' req.params = params memcache = cache_from_env(req.environ, True) if (req.method == 'GET' and record_type != 'object' and self.app.recheck_listing_shard_ranges > 0 and memcache and get_param(req, 'states') == 'listing' and not config_true_value( req.headers.get('x-backend-include-deleted', False))): # This GET might be served from cache or might populate cache. # 'x-backend-include-deleted' is not usually expected in requests # to the proxy (it is used from sharder to container servers) but # it is included in the conditions just in case because we don't # cache deleted shard ranges. resp = self._GET_using_cache(req) else: resp = self._GETorHEAD_from_backend(req) resp_record_type = resp.headers.get('X-Backend-Record-Type', '') if all((req.method == "GET", record_type == 'auto', resp_record_type.lower() == 'shard')): resp = self._get_from_shards(req, resp) if not config_true_value(resp.headers.get('X-Backend-Cached-Results')): # Cache container metadata. We just made a request to a storage # node and got up-to-date information for the container. resp.headers['X-Backend-Recheck-Container-Existence'] = str( self.app.recheck_container_existence) set_info_cache(self.app, req.environ, self.account_name, self.container_name, resp) if 'swift.authorize' in req.environ: req.acl = resp.headers.get('x-container-read') aresp = req.environ['swift.authorize'](req) if aresp: # Don't cache this. It doesn't reflect the state of the # container, just that the user can't access it. return aresp if not req.environ.get('swift_owner', False): for key in self.app.swift_owner_headers: if key in resp.headers: del resp.headers[key] # Expose sharding state in reseller requests if req.environ.get('reseller_request', False): resp.headers['X-Container-Sharding'] = config_true_value( resp.headers.get( get_sys_meta_prefix('container') + 'Sharding', 'False')) return resp
* Move to separate account, not container, for trash. This requires Swift to allow cross-account COPY requests. * If block_trash_deletes is on, modify the Allow header in responses (both OPTIONS responses and any other 405 response). """ from swift.common import http, swob, utils, wsgi from swift.common.request_helpers import get_sys_meta_prefix from swift.proxy.controllers.base import get_account_info, get_container_info DEFAULT_TRASH_PREFIX = ".trash-" DEFAULT_TRASH_LIFETIME = 86400 * 90 # 90 days expressed in seconds SYSMETA_UNDELETE_ENABLED = "undelete-enabled" SYSMETA_ACCOUNT = get_sys_meta_prefix('account') + SYSMETA_UNDELETE_ENABLED SYSMETA_CONTAINER = get_sys_meta_prefix('container') + SYSMETA_UNDELETE_ENABLED try: from swift.common.request_helpers import close_if_possible except ImportError: # Pre-1.13.0 (ref. https://github.com/openstack/swift/commit/1f67eb7) def close_if_possible(maybe_closable): close_method = getattr(maybe_closable, 'close', None) if callable(close_method): return close_method() def friendly_error(orig_error): return "Error copying object to trash:\n" + orig_error
# not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from builtins import object from swift.common.request_helpers import get_sys_meta_prefix from swift.common.swob import Request SYSMETA_HEADER = get_sys_meta_prefix('account') + 'project-domain-id' OVERRIDE_HEADER = 'x-account-project-domain-id-override' class DomainOverrideMiddleware(object): def __init__(self, app, **kwargs): self.app = app @classmethod def factory(cls, global_config, **local_config): def _factory(app): return cls(app, **local_config) return _factory def __call__(self, environ, start_response): # only reseller may invoke the override
""" from swift.common.swob import wsgify from swift.common.utils import get_logger from swift.common.request_helpers import remove_items, get_sys_meta_prefix import re #: A list of python regular expressions that will be used to #: match against inbound request headers. Matching headers will #: be removed from the request. # Exclude headers starting with a sysmeta prefix. # If adding to this list, note that these are regex patterns, # so use a trailing $ to constrain to an exact header match # rather than prefix match. inbound_exclusions = [ get_sys_meta_prefix('account'), get_sys_meta_prefix('container'), get_sys_meta_prefix('object') ] # 'x-object-sysmeta' is reserved in anticipation of future support # for system metadata being applied to objects #: A list of python regular expressions that will be used to #: match against outbound response headers. Matching headers will #: be removed from the response. outbound_exclusions = inbound_exclusions def make_exclusion_test(exclusions): expr = '|'.join(exclusions) test = re.compile(expr, re.IGNORECASE)
from swift.common.utils import register_swift_info from swift.common.utils import split_path from swift.proxy.controllers.base import get_container_info from eventlet.green import urllib2 from eventlet import Timeout import base64 import json import ssl import urlparse # Container system metadata # x-container-sysmeta-webhookk & x-container-sysmeta-webhook-auth # Add x-container-sysmeta-webhook-get for hooks called during get requests SYSMETA_WEBHOOK = get_sys_meta_prefix('container') + 'webhook' SYSMETA_WEBHOOK_AUTH = get_sys_meta_prefix('container') + 'webhook-auth' SYSMETA_WEBHOOK_GET = get_sys_meta_prefix('container') + 'webhook-get' SYSMETA_WEBHOOK_GET_AUTH = get_sys_meta_prefix( 'container') + 'webhook-get-auth' class WebhookMiddleware(object): def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='webhook') @wsgify def __call__(self, req): obj = None container = None
MD5_OF_EMPTY_STRING, closing_if_possible from swift.common.constraints import check_account_format from swift.common.wsgi import WSGIContext, make_subrequest from swift.common.request_helpers import get_sys_meta_prefix, \ check_path_header from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \ HTTPException, HTTPConflict, HTTPPreconditionFailed from swift.common.http import is_success from swift.common.exceptions import LinkIterError from swift.common.header_key_dict import HeaderKeyDict DEFAULT_SYMLOOP_MAX = 2 # Header values for symlink target path strings will be quoted values. TGT_OBJ_SYMLINK_HDR = 'x-symlink-target' TGT_ACCT_SYMLINK_HDR = 'x-symlink-target-account' TGT_OBJ_SYSMETA_SYMLINK_HDR = get_sys_meta_prefix('object') + 'symlink-target' TGT_ACCT_SYSMETA_SYMLINK_HDR = \ get_sys_meta_prefix('object') + 'symlink-target-account' def _check_symlink_header(req): """ Validate that the value from x-symlink-target header is well formatted. We assume the caller ensures that x-symlink-target header is present in req.headers. :param req: HTTP request object :raise: HTTPPreconditionFailed if x-symlink-target value is not well formatted. :raise: HTTPBadRequest if the x-symlink-target value points to the request path.
def purge_crypto_sysmeta_headers(headers): return [h for h in headers if not h[0].lower().startswith( (get_object_transient_sysmeta('crypto-'), get_sys_meta_prefix('object') + 'crypto-'))]
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from swift.common import utils as swift_utils from swift.common.http import is_success from swift.common.middleware import acl as swift_acl from swift.common.request_helpers import get_sys_meta_prefix from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized from swift.common.utils import config_read_reseller_options, list_from_csv from swift.proxy.controllers.base import get_account_info import functools PROJECT_DOMAIN_ID_HEADER = 'x-account-project-domain-id' PROJECT_DOMAIN_ID_SYSMETA_HEADER = \ get_sys_meta_prefix('account') + 'project-domain-id' # a string that is unique w.r.t valid ids UNKNOWN_ID = '_unknown' class KeystoneAuth(object): """Swift middleware to Keystone authorization system. In Swift's proxy-server.conf add this keystoneauth middleware and the authtoken middleware to your pipeline. Make sure you have the authtoken middleware before the keystoneauth middleware. The authtoken middleware will take care of validating the user and keystoneauth will authorize access. The sample proxy-server.conf shows a sample pipeline that uses keystone.
URL = "http://141.115.103.32:8081" ENDPOINT_PATH = "/api/experimental" DAG_NAME = "test" # # import ConfigParser # config = ConfigParser.ConfigParser().read(CONFIG_PATH).sections() import six if six.PY3: from eventlet.green.urllib import request as urllib2 else: from eventlet.green import urllib2 import requests # x-container-sysmeta-webhook SYSMETA_WEBHOOK = get_sys_meta_prefix('container') + 'webhook' import json class NewDataTriggerMiddleware(object): def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='newdatatrigger') @wsgify def __call__(self, req): print(req) print(req.headers) print(req.path_info) obj = None try: