def create_container(self, req, container_path): """ Checks if the container exists and if not try to create it. :params container_path: an unquoted path to a container to be created :returns: True if created container, False if container exists :raises CreateContainerError: when unable to create container """ head_cont_req = make_subrequest( req.environ, method='HEAD', path=wsgi_quote(container_path), headers={'X-Auth-Token': req.headers.get('X-Auth-Token')}, swift_source='EA') resp = head_cont_req.get_response(self.app) if resp.is_success: return False if resp.status_int == HTTP_NOT_FOUND: create_cont_req = make_subrequest( req.environ, method='PUT', path=wsgi_quote(container_path), headers={'X-Auth-Token': req.headers.get('X-Auth-Token')}, swift_source='EA') resp = create_cont_req.get_response(self.app) if resp.is_success: return True raise CreateContainerError( "Create Container Failed: " + container_path, resp.status_int, resp.status)
def test_check_symlink_header(self): def do_test(headers): req = Request.blank('/v1/a/c/o', method='PUT', headers=headers) symlink._check_symlink_header(req) # normal cases do_test({'X-Symlink-Target': 'c1/o1'}) do_test({'X-Symlink-Target': 'c1/sub/o1'}) do_test({'X-Symlink-Target': 'c1%2Fo1'}) # specify account do_test({'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': 'another'}) # URL encoded is safe do_test({'X-Symlink-Target': 'c1%2Fo1'}) # URL encoded + multibytes is also safe target = u'\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' target = swob.bytes_to_wsgi(target.encode('utf8')) do_test({'X-Symlink-Target': target}) do_test({'X-Symlink-Target': swob.wsgi_quote(target)}) target = swob.bytes_to_wsgi(u'\u30b0\u30e9\u30d6\u30eb'.encode('utf8')) do_test( {'X-Symlink-Target': 'cont/obj', 'X-Symlink-Target-Account': target}) do_test( {'X-Symlink-Target': 'cont/obj', 'X-Symlink-Target-Account': swob.wsgi_quote(target)})
def _validate_and_prep_request_headers(req): """ Validate that the value from x-symlink-target header is well formatted and that the x-symlink-target-etag header (if present) does not contain problematic characters. We assume the caller ensures that x-symlink-target header is present in req.headers. :param req: HTTP request object :returns: a tuple, the full versioned path to the object (as a WSGI string) and the X-Symlink-Target-Etag header value which may be None :raise: HTTPPreconditionFailed if x-symlink-target value is not well formatted. :raise: HTTPBadRequest if the x-symlink-target value points to the request path. :raise: HTTPBadRequest if the x-symlink-target-etag value contains a semicolon, double-quote, or backslash. """ # N.B. check_path_header doesn't assert the leading slash and # copy middleware may accept the format. In the symlink, API # says apparently to use "container/object" format so add the # validation first, here. error_body = 'X-Symlink-Target header must be of the form ' \ '<container name>/<object name>' if wsgi_unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'): raise HTTPPreconditionFailed(body=error_body, request=req, content_type='text/plain') # check container and object format container, obj = check_path_header(req, TGT_OBJ_SYMLINK_HDR, 2, error_body) req.headers[TGT_OBJ_SYMLINK_HDR] = wsgi_quote('%s/%s' % (container, obj)) # Check account format if it exists account = check_account_format( req, wsgi_unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \ if TGT_ACCT_SYMLINK_HDR in req.headers else None # Extract request path _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True) if account: req.headers[TGT_ACCT_SYMLINK_HDR] = wsgi_quote(account) else: account = req_acc # Check if symlink targets the symlink itself or not if (account, container, obj) == (req_acc, req_cont, req_obj): raise HTTPBadRequest(body='Symlink cannot target itself', request=req, content_type='text/plain') etag = req.headers.get(TGT_ETAG_SYMLINK_HDR, None) if etag and any(c in etag for c in ';"\\'): # See cgi.parse_header for why the above chars are problematic raise HTTPBadRequest(body='Bad %s format' % TGT_ETAG_SYMLINK_HDR.title(), request=req, content_type='text/plain') if not (etag or req.headers.get('Content-Type')): req.headers['Content-Type'] = 'application/symlink' return '/v1/%s/%s/%s' % (account, container, obj), etag
def create_container(self, req, container_path): """ Checks if the container exists and if not try to create it. :params container_path: an unquoted path to a container to be created :returns: True if created container, False if container exists :raises CreateContainerError: when unable to create container """ head_cont_req = make_subrequest( req.environ, method='HEAD', path=wsgi_quote(container_path), headers={'X-Auth-Token': req.headers.get('X-Auth-Token')}, swift_source='EA') resp = head_cont_req.get_response(self.app) if resp.is_success: return False if resp.status_int == HTTP_NOT_FOUND: create_cont_req = make_subrequest( req.environ, method='PUT', path=wsgi_quote(container_path), headers={'X-Auth-Token': req.headers.get('X-Auth-Token')}, swift_source='EA') resp = create_cont_req.get_response(self.app) if resp.is_success: return True raise CreateContainerError( "Create Container Failed: " + container_path, resp.status_int, resp.status)
def delete_filter(predicate, objs_to_delete): for obj_to_delete in objs_to_delete: obj_name = obj_to_delete['name'] if not obj_name: continue if not predicate(obj_name): continue if obj_to_delete.get('error'): if obj_to_delete['error']['code'] == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 else: failed_files.append([ wsgi_quote(str_to_wsgi(obj_name)), obj_to_delete['error']['message'] ]) continue delete_path = '/'.join( ['', vrs, account, obj_name.lstrip('/')]) if not constraints.check_utf8(delete_path): failed_files.append([ wsgi_quote(str_to_wsgi(obj_name)), HTTPPreconditionFailed().status ]) continue yield (obj_name, delete_path, obj_to_delete.get('version_id'))
def handle_COPY(self, req, start_response, account, container, obj): if not req.headers.get('Destination'): return HTTPPreconditionFailed(request=req, body='Destination header required' )(req.environ, start_response) dest_account = account if 'Destination-Account' in req.headers: dest_account = wsgi_unquote(req.headers.get('Destination-Account')) dest_account = check_account_format(req, dest_account) req.headers['X-Copy-From-Account'] = wsgi_quote(account) account = dest_account del req.headers['Destination-Account'] dest_container, dest_object = _check_destination_header(req) source = '/%s/%s' % (container, obj) container = dest_container obj = dest_object # re-write the existing request as a PUT instead of creating a new one req.method = 'PUT' # As this the path info is updated with destination container, # the proxy server app will use the right object controller # implementation corresponding to the container's policy type. ver, _junk = req.split_path(1, 2, rest_with_last=True) req.path_info = '/%s/%s/%s/%s' % ( ver, dest_account, dest_container, dest_object) req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = wsgi_quote(source) del req.headers['Destination'] return self.handle_PUT(req, start_response)
def _process_delete(self, resp, pile, obj_name, resp_dict, failed_files, failed_file_response, retry=0): if resp.status_int // 100 == 2: resp_dict['Number Deleted'] += 1 elif resp.status_int == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 elif resp.status_int == HTTP_UNAUTHORIZED: failed_files.append( [wsgi_quote(str_to_wsgi(obj_name)), HTTPUnauthorized().status]) elif resp.status_int == HTTP_CONFLICT and pile and \ self.retry_count > 0 and self.retry_count > retry: retry += 1 sleep(self.retry_interval**retry) delete_obj_req = Request.blank(resp.environ['PATH_INFO'], resp.environ) def _retry(req, app, obj_name, retry): return req.get_response(app), obj_name, retry pile.spawn(_retry, delete_obj_req, self.app, obj_name, retry) else: if resp.status_int // 100 == 5: failed_file_response['type'] = HTTPBadGateway failed_files.append( [wsgi_quote(str_to_wsgi(obj_name)), resp.status])
def handle_COPY(self, req, start_response, account, container, obj): if not req.headers.get('Destination'): return HTTPPreconditionFailed(request=req, body='Destination header required')( req.environ, start_response) dest_account = account if 'Destination-Account' in req.headers: dest_account = wsgi_unquote(req.headers.get('Destination-Account')) dest_account = check_account_format(req, dest_account) req.headers['X-Copy-From-Account'] = wsgi_quote(account) account = dest_account del req.headers['Destination-Account'] dest_container, dest_object = _check_destination_header(req) source = '/%s/%s' % (container, obj) container = dest_container obj = dest_object # re-write the existing request as a PUT instead of creating a new one req.method = 'PUT' # As this the path info is updated with destination container, # the proxy server app will use the right object controller # implementation corresponding to the container's policy type. ver, _junk = req.split_path(1, 2, rest_with_last=True) req.path_info = '/%s/%s/%s/%s' % (ver, dest_account, dest_container, dest_object) req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = wsgi_quote(source) del req.headers['Destination'] return self.handle_PUT(req, start_response)
def _listing_pages_iter(self, account_name, lcontainer, lprefix, req, marker='', end_marker='', reverse=True): '''Get "pages" worth of objects that start with a prefix. The optional keyword arguments ``marker``, ``end_marker``, and ``reverse`` are used similar to how they are for containers. We're either coming: - directly from ``_listing_iter``, in which case none of the optional args are specified, or - from ``_in_proxy_reverse_listing``, in which case ``reverse`` is ``False`` and both ``marker`` and ``end_marker`` are specified (although they may still be blank). ''' while True: lreq = make_pre_authed_request( req.environ, method='GET', swift_source='VW', path=wsgi_quote('/v1/%s/%s' % (account_name, lcontainer))) lreq.environ['QUERY_STRING'] = \ 'prefix=%s&marker=%s' % (wsgi_quote(lprefix), wsgi_quote(marker)) if end_marker: lreq.environ['QUERY_STRING'] += '&end_marker=%s' % ( wsgi_quote(end_marker)) if reverse: lreq.environ['QUERY_STRING'] += '&reverse=on' lresp = lreq.get_response(self.app) if not is_success(lresp.status_int): close_if_possible(lresp.app_iter) if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif is_client_error(lresp.status_int): raise HTTPPreconditionFailed(request=req) else: raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break # When using the ``reverse`` param, check that the listing is # actually reversed first_item = bytes_to_wsgi(sublisting[0]['name'].encode('utf-8')) last_item = bytes_to_wsgi(sublisting[-1]['name'].encode('utf-8')) page_is_after_marker = marker and first_item > marker if reverse and (first_item < last_item or page_is_after_marker): # Apparently there's at least one pre-2.6.0 container server yield self._in_proxy_reverse_listing( account_name, lcontainer, lprefix, req, marker, sublisting) return marker = last_item yield sublisting
def _listing_pages_iter(self, account_name, lcontainer, lprefix, req, marker='', end_marker='', reverse=True): '''Get "pages" worth of objects that start with a prefix. The optional keyword arguments ``marker``, ``end_marker``, and ``reverse`` are used similar to how they are for containers. We're either coming: - directly from ``_listing_iter``, in which case none of the optional args are specified, or - from ``_in_proxy_reverse_listing``, in which case ``reverse`` is ``False`` and both ``marker`` and ``end_marker`` are specified (although they may still be blank). ''' while True: lreq = make_pre_authed_request( req.environ, method='GET', swift_source='VW', path=wsgi_quote('/v1/%s/%s' % (account_name, lcontainer))) lreq.environ['QUERY_STRING'] = \ 'prefix=%s&marker=%s' % (wsgi_quote(lprefix), wsgi_quote(marker)) if end_marker: lreq.environ['QUERY_STRING'] += '&end_marker=%s' % ( wsgi_quote(end_marker)) if reverse: lreq.environ['QUERY_STRING'] += '&reverse=on' lresp = lreq.get_response(self.app) if not is_success(lresp.status_int): close_if_possible(lresp.app_iter) if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif is_client_error(lresp.status_int): raise HTTPPreconditionFailed(request=req) else: raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break # When using the ``reverse`` param, check that the listing is # actually reversed first_item = bytes_to_wsgi(sublisting[0]['name'].encode('utf-8')) last_item = bytes_to_wsgi(sublisting[-1]['name'].encode('utf-8')) page_is_after_marker = marker and first_item > marker if reverse and (first_item < last_item or page_is_after_marker): # Apparently there's at least one pre-2.6.0 container server yield self._in_proxy_reverse_listing( account_name, lcontainer, lprefix, req, marker, sublisting) return marker = last_item yield sublisting
def _check_symlink_header(req): """ Validate that the value from x-symlink-target header is well formatted. We assume the caller ensures that x-symlink-target header is present in req.headers. :param req: HTTP request object :raise: HTTPPreconditionFailed if x-symlink-target value is not well formatted. :raise: HTTPBadRequest if the x-symlink-target value points to the request path. """ # N.B. check_path_header doesn't assert the leading slash and # copy middleware may accept the format. In the symlink, API # says apparently to use "container/object" format so add the # validation first, here. error_body = 'X-Symlink-Target header must be of the form ' \ '<container name>/<object name>' try: if wsgi_unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'): raise HTTPPreconditionFailed(body=error_body, request=req, content_type='text/plain') except TypeError: raise HTTPPreconditionFailed(body=error_body, request=req, content_type='text/plain') # check container and object format container, obj = check_path_header(req, TGT_OBJ_SYMLINK_HDR, 2, error_body) req.headers[TGT_OBJ_SYMLINK_HDR] = wsgi_quote('%s/%s' % (container, obj)) # Check account format if it exists try: account = check_account_format( req, wsgi_unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \ if TGT_ACCT_SYMLINK_HDR in req.headers else None except TypeError: raise HTTPPreconditionFailed( body='Account name cannot contain slashes', request=req, content_type='text/plain') # Extract request path _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True) if account: req.headers[TGT_ACCT_SYMLINK_HDR] = wsgi_quote(account) else: account = req_acc # Check if symlink targets the symlink itself or not if (account, container, obj) == (req_acc, req_cont, req_obj): raise HTTPBadRequest(body='Symlink cannot target itself', request=req, content_type='text/plain')
def _create_response_headers(self, source_path, source_resp, sink_req): resp_headers = dict() acct, path = source_path.split('/', 3)[2:4] resp_headers['X-Copied-From-Account'] = wsgi_quote(acct) resp_headers['X-Copied-From'] = wsgi_quote(path) if 'last-modified' in source_resp.headers: resp_headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] # Existing sys and user meta of source object is added to response # headers in addition to the new ones. _copy_headers(sink_req.headers, resp_headers) return resp_headers
def _create_response_headers(self, source_path, source_resp, sink_req): resp_headers = dict() acct, path = source_path.split('/', 3)[2:4] resp_headers['X-Copied-From-Account'] = wsgi_quote(acct) resp_headers['X-Copied-From'] = wsgi_quote(path) if 'last-modified' in source_resp.headers: resp_headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] # Existing sys and user meta of source object is added to response # headers in addition to the new ones. _copy_headers(sink_req.headers, resp_headers) return resp_headers
def _get_container_listing(self, req, version, account, container, prefix, marker=''): ''' :param version: whatever :param account: native :param container: native :param prefix: native :param marker: native ''' con_req = make_subrequest( req.environ, path=wsgi_quote('/'.join([ '', str_to_wsgi(version), str_to_wsgi(account), str_to_wsgi(container)])), method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO') con_req.query_string = 'prefix=%s' % quote(prefix) if marker: con_req.query_string += '&marker=%s' % quote(marker) con_resp = con_req.get_response(self.dlo.app) if not is_success(con_resp.status_int): if req.method == 'HEAD': con_resp.body = b'' return con_resp, None with closing_if_possible(con_resp.app_iter): return None, json.loads(b''.join(con_resp.app_iter))
def build_traversal_req(symlink_target): """ :returns: new request for target path if it's symlink otherwise None """ version, account, _junk = req.split_path(2, 3, True) account = self._response_header_value( TGT_ACCT_SYSMETA_SYMLINK_HDR) or wsgi_quote(account) target_path = os.path.join('/', version, account, symlink_target.lstrip('/')) self._last_target_path = target_path subreq_headers = dict(req.headers) if self._response_header_value(ALLOW_RESERVED_NAMES): # this symlink's sysmeta says it can point to reserved names, # we're infering that some piece of middleware had previously # authorized this request because users can't access reserved # names directly subreq_meth = make_pre_authed_request subreq_headers['X-Backend-Allow-Reserved-Names'] = 'true' else: subreq_meth = make_subrequest new_req = subreq_meth(orig_req.environ, path=target_path, method=req.method, headers=subreq_headers, swift_source='SYM') new_req.headers.pop('X-Backend-Storage-Policy-Index', None) return new_req
def delete_actual_object(self, actual_obj, timestamp, is_async_delete): """ Deletes the end-user object indicated by the actual object name given '<account>/<container>/<object>' if and only if the X-Delete-At value of the object is exactly the timestamp given. :param actual_obj: The name of the end-user object to delete: '<account>/<container>/<object>' :param timestamp: The swift.common.utils.Timestamp instance the X-Delete-At value must match to perform the actual delete. :param is_async_delete: False if the object should be deleted because of "normal" expiration, or True if it should be async-deleted. :raises UnexpectedResponse: if the delete was unsuccessful and should be retried later """ path = '/v1/' + wsgi_quote(str_to_wsgi(actual_obj.lstrip('/'))) if is_async_delete: headers = {'X-Timestamp': timestamp.normal} acceptable_statuses = (2, HTTP_CONFLICT, HTTP_NOT_FOUND) else: headers = { 'X-Timestamp': timestamp.normal, 'X-If-Delete-At': timestamp.normal, 'X-Backend-Clean-Expiring-Object-Queue': 'no' } acceptable_statuses = (2, HTTP_CONFLICT) self.swift.make_request('DELETE', path, headers, acceptable_statuses)
def delete_actual_object(self, actual_obj, timestamp, is_async_delete): """ Deletes the end-user object indicated by the actual object name given '<account>/<container>/<object>' if and only if the X-Delete-At value of the object is exactly the timestamp given. :param actual_obj: The name of the end-user object to delete: '<account>/<container>/<object>' :param timestamp: The swift.common.utils.Timestamp instance the X-Delete-At value must match to perform the actual delete. :param is_async_delete: False if the object should be deleted because of "normal" expiration, or True if it should be async-deleted. :raises UnexpectedResponse: if the delete was unsuccessful and should be retried later """ path = '/v1/' + wsgi_quote(str_to_wsgi(actual_obj.lstrip('/'))) if is_async_delete: headers = {'X-Timestamp': timestamp.normal} acceptable_statuses = (2, HTTP_CONFLICT, HTTP_NOT_FOUND) else: headers = {'X-Timestamp': timestamp.normal, 'X-If-Delete-At': timestamp.normal, 'X-Backend-Clean-Expiring-Object-Queue': 'no'} acceptable_statuses = (2, HTTP_CONFLICT) self.swift.make_request('DELETE', path, headers, acceptable_statuses)
def _get_container_listing(self, req, version, account, container, prefix, marker=''): ''' :param version: whatever :param account: native :param container: native :param prefix: native :param marker: native ''' con_req = make_subrequest( req.environ, path=wsgi_quote('/'.join([ '', str_to_wsgi(version), str_to_wsgi(account), str_to_wsgi(container)])), method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO') con_req.query_string = 'prefix=%s' % quote(prefix) if marker: con_req.query_string += '&marker=%s' % quote(marker) con_resp = con_req.get_response(self.dlo.app) if not is_success(con_resp.status_int): if req.method == 'HEAD': con_resp.body = b'' return con_resp, None with closing_if_possible(con_resp.app_iter): return None, json.loads(b''.join(con_resp.app_iter))
def parse_request(self): if not six.PY2: # request lines *should* be ascii per the RFC, but historically # we've allowed (and even have func tests that use) arbitrary # bytes. This breaks on py3 (see https://bugs.python.org/issue33973 # ) but the work-around is simple: munge the request line to be # properly quoted. py2 will do the right thing without this, but it # doesn't hurt to re-write the request line like this and it # simplifies testing. if self.raw_requestline.count(b' ') >= 2: parts = self.raw_requestline.split(b' ', 2) path, q, query = parts[1].partition(b'?') # unquote first, so we don't over-quote something # that was *correctly* quoted path = wsgi_to_bytes(wsgi_quote(wsgi_unquote( bytes_to_wsgi(path)))) query = b'&'.join( sep.join([ wsgi_to_bytes(wsgi_quote_plus(wsgi_unquote_plus( bytes_to_wsgi(key)))), wsgi_to_bytes(wsgi_quote_plus(wsgi_unquote_plus( bytes_to_wsgi(val)))) ]) for part in query.split(b'&') for key, sep, val in (part.partition(b'='), )) parts[1] = path + q + query self.raw_requestline = b' '.join(parts) # else, mangled protocol, most likely; let base class deal with it return wsgi.HttpProtocol.parse_request(self)
def do_delete(obj_name, delete_path): delete_obj_req = make_subrequest( req.environ, method='DELETE', path=wsgi_quote(str_to_wsgi(delete_path)), headers={'X-Auth-Token': req.headers.get('X-Auth-Token')}, body='', agent='%(orig)s ' + user_agent, swift_source=swift_source) return (delete_obj_req.get_response(self.app), obj_name, 0)
def _redirect_with_slash(self, env_, start_response): env = {} env.update(env_) if self.url_scheme: env['wsgi.url_scheme'] = self.url_scheme if self.url_host: env['HTTP_HOST'] = self.url_host resp = HTTPMovedPermanently(location=wsgi_quote(env['PATH_INFO'] + '/')) return resp(env, start_response)
def _redirect_with_slash(self, env_, start_response): env = {} env.update(env_) if self.url_scheme: env['wsgi.url_scheme'] = self.url_scheme if self.url_host: env['HTTP_HOST'] = self.url_host resp = HTTPMovedPermanently( location=wsgi_quote(env['PATH_INFO'] + '/')) return resp(env, start_response)
def do_delete(obj_name, delete_path): delete_obj_req = make_subrequest( req.environ, method='DELETE', path=wsgi_quote(str_to_wsgi(delete_path)), headers={'X-Auth-Token': req.headers.get('X-Auth-Token')}, body='', agent='%(orig)s ' + user_agent, swift_source=swift_source) return (delete_obj_req.get_response(self.app), obj_name, 0)
def handle_request(self, req, versions_cont, api_version, account, container, obj, is_enabled): if req.method == 'PUT': return self.handle_put(req, versions_cont, api_version, account, obj, is_enabled) elif req.method == 'POST': return self.handle_post(req, versions_cont, account) elif req.method == 'DELETE': return self.handle_delete(req, versions_cont, api_version, account, container, obj, is_enabled) # GET/HEAD/OPTIONS resp = req.get_response(self.app) resp.headers['X-Object-Version-Id'] = resp.headers[VERSION_ID_HEADER] # Check for a "real" version loc = wsgi_unquote(resp.headers.get('Content-Location', '')) if loc: _, acct, cont, version_obj = split_path(loc, 4, 4, True) if acct == account and cont == versions_cont: _, version = self._split_version_from_name(version_obj) if version is not None: resp.headers['X-Object-Version-Id'] = version.internal content_loc = wsgi_quote('/%s/%s/%s/%s' % ( api_version, account, container, obj, )) + '?version-id=%s' % (version.internal, ) resp.headers['Content-Location'] = content_loc symlink_target = wsgi_unquote(resp.headers.get('X-Symlink-Target', '')) if symlink_target: cont, version_obj = split_path('/%s' % symlink_target, 2, 2, True) if cont == versions_cont: _, version = self._split_version_from_name(version_obj) if version is not None: resp.headers['X-Object-Version-Id'] = version.internal symlink_target = wsgi_quote('%s/%s' % (container, obj)) + \ '?version-id=%s' % (version.internal,) resp.headers['X-Symlink-Target'] = symlink_target return resp
def _put_versioned_obj(self, req, put_path_info, source_resp): # Create a new Request object to PUT to the versions container, copying # all headers from the source object apart from x-timestamp. put_req = make_pre_authed_request( req.environ, path=wsgi_quote(put_path_info), method='PUT', swift_source='VW') copy_header_subset(source_resp, put_req, lambda k: k.lower() != 'x-timestamp') put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) put_resp = put_req.get_response(self.app) close_if_possible(source_resp.app_iter) return put_resp
def handle_delete(self, req, start_response): """ Handle request to delete a user's container. As part of deleting a container, this middleware will also delete the hidden container holding object versions. Before a user's container can be deleted, swift must check if there are still old object versions from that container. Only after disabling versioning and deleting *all* object versions can a container be deleted. """ container_info = get_container_info(req.environ, self.app, swift_source='OV') versions_cont = unquote( container_info.get('sysmeta', {}).get('versions-container', '')) if versions_cont: account = req.split_path(3, 3, True)[1] # using a HEAD request here as opposed to get_container_info # to make sure we get an up-to-date value versions_req = make_pre_authed_request( req.environ, method='HEAD', swift_source='OV', path=wsgi_quote('/v1/%s/%s' % (account, str_to_wsgi(versions_cont))), headers={'X-Backend-Allow-Reserved-Names': 'true'}) vresp = versions_req.get_response(self.app) drain_and_close(vresp) if vresp.is_success and int( vresp.headers.get('X-Container-Object-Count', 0)) > 0: raise HTTPConflict( 'Delete all versions before deleting container.', request=req) elif not vresp.is_success and vresp.status_int != 404: raise HTTPInternalServerError( 'Error deleting versioned container') else: versions_req.method = 'DELETE' resp = versions_req.get_response(self.app) drain_and_close(resp) if not is_success(resp.status_int) and resp.status_int != 404: raise HTTPInternalServerError( 'Error deleting versioned container') app_resp = self._app_call(req.environ) start_response(self._response_status, self._response_headers, self._response_exc_info) return app_resp
def _put_versioned_obj(self, req, put_path_info, source_resp): # Create a new Request object to PUT to the versions container, copying # all headers from the source object apart from x-timestamp. put_req = make_pre_authed_request( req.environ, path=wsgi_quote(put_path_info), method='PUT', swift_source='VW') copy_header_subset(source_resp, put_req, lambda k: k.lower() != 'x-timestamp') put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) put_resp = put_req.get_response(self.app) close_if_possible(source_resp.app_iter) return put_resp
def _validate_etag_and_update_sysmeta(self, req, symlink_target_path, etag): if req.environ.get('swift.symlink_override'): req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \ req.headers[TGT_BYTES_SYMLINK_HDR] return # next we'll make sure the E-Tag matches a real object new_req = make_subrequest(req.environ, path=wsgi_quote(symlink_target_path), method='HEAD', swift_source='SYM') if req.allow_reserved_names: new_req.headers['X-Backend-Allow-Reserved-Names'] = 'true' self._last_target_path = symlink_target_path resp = self._recursive_get_head(new_req, target_etag=etag, follow_softlinks=False) if self._get_status_int() == HTTP_NOT_FOUND: raise HTTPConflict(body='X-Symlink-Target does not exist', request=req, headers={ 'Content-Type': 'text/plain', 'Content-Location': self._last_target_path }) if not is_success(self._get_status_int()): drain_and_close(resp) raise status_map[self._get_status_int()](request=req) response_headers = HeaderKeyDict(self._response_headers) # carry forward any etag update params (e.g. "slo_etag"), we'll append # symlink_target_* params to this header after this method returns override_header = get_container_update_override_key('etag') if override_header in response_headers and \ override_header not in req.headers: sep, params = response_headers[override_header].partition(';')[1:] req.headers[override_header] = MD5_OF_EMPTY_STRING + sep + params # It's troublesome that there's so much leakage with SLO if 'X-Object-Sysmeta-Slo-Etag' in response_headers and \ override_header not in req.headers: req.headers[override_header] = '%s; slo_etag=%s' % ( MD5_OF_EMPTY_STRING, response_headers['X-Object-Sysmeta-Slo-Etag']) req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = ( response_headers.get('x-object-sysmeta-slo-size') or response_headers['Content-Length']) req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag if not req.headers.get('Content-Type'): req.headers['Content-Type'] = response_headers['Content-Type']
def filter_reserved(self, listing, account, container): new_listing = [] for entry in list(listing): for key in ('name', 'subdir'): value = entry.get(key, '') if six.PY2: value = value.encode('utf-8') if RESERVED in value: if container: self.logger.warning( 'Container listing for %s/%s had ' 'reserved byte in %s: %r', wsgi_quote(account), wsgi_quote(container), key, value) else: self.logger.warning( 'Account listing for %s had ' 'reserved byte in %s: %r', wsgi_quote(account), key, value) break # out of the *key* loop; check next entry else: new_listing.append(entry) return new_listing
def delete_filter(predicate, objs_to_delete): for obj_to_delete in objs_to_delete: obj_name = obj_to_delete['name'] if not obj_name: continue if not predicate(obj_name): continue if obj_to_delete.get('error'): if obj_to_delete['error']['code'] == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 else: failed_files.append([ wsgi_quote(str_to_wsgi(obj_name)), obj_to_delete['error']['message']]) continue delete_path = '/'.join(['', vrs, account, obj_name.lstrip('/')]) if not constraints.check_utf8(delete_path): failed_files.append([wsgi_quote(str_to_wsgi(obj_name)), HTTPPreconditionFailed().status]) continue yield (obj_name, delete_path)
def handle_post(self, req): """ Handle post request. If POSTing to a symlink, a HTTPTemporaryRedirect error message is returned to client. Clients that POST to symlinks should understand that the POST is not redirected to the target object like in a HEAD/GET request. POSTs to a symlink will be handled just like a normal object by the object server. It cannot reject it because it may not have symlink state when the POST lands. The object server has no knowledge of what is a symlink object is. On the other hand, on POST requests, the object server returns all sysmeta of the object. This method uses that sysmeta to determine if the stored object is a symlink or not. :param req: HTTP POST object request :raises: HTTPTemporaryRedirect if POSTing to a symlink. :returns: Response Iterator """ if TGT_OBJ_SYMLINK_HDR in req.headers: raise HTTPBadRequest( body='A PUT request is required to set a symlink target', request=req, content_type='text/plain') resp = self._app_call(req.environ) if not is_success(self._get_status_int()): return resp tgt_co = self._response_header_value(TGT_OBJ_SYSMETA_SYMLINK_HDR) if tgt_co: version, account, _junk = req.split_path(2, 3, True) target_acc = self._response_header_value( TGT_ACCT_SYSMETA_SYMLINK_HDR) or wsgi_quote(account) location_hdr = os.path.join( '/', version, target_acc, tgt_co) headers = {'location': location_hdr} tgt_etag = self._response_header_value( TGT_ETAG_SYSMETA_SYMLINK_HDR) if tgt_etag: headers[TGT_ETAG_SYMLINK_HDR] = tgt_etag req.environ['swift.leave_relative_location'] = True errmsg = 'The requested POST was applied to a symlink. POST ' +\ 'directly to the target to apply requested metadata.' for key, value in self._response_headers: if key.lower().startswith('x-object-sysmeta-'): headers[key] = value raise HTTPTemporaryRedirect( body=errmsg, headers=headers) else: return resp
def _get_source_object(self, req, path_info): # make a pre_auth request in case the user has write access # to container, but not READ. This was allowed in previous version # (i.e., before middleware) so keeping the same behavior here get_req = make_pre_authed_request( req.environ, path=wsgi_quote(path_info), headers={'X-Newest': 'True'}, method='GET', swift_source='VW') source_resp = get_req.get_response(self.app) if source_resp.content_length is None or \ source_resp.content_length > MAX_FILE_SIZE: close_if_possible(source_resp.app_iter) return HTTPRequestEntityTooLarge(request=req) return source_resp
def _process_delete(self, resp, pile, obj_name, resp_dict, failed_files, failed_file_response, retry=0): if resp.status_int // 100 == 2: resp_dict['Number Deleted'] += 1 elif resp.status_int == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 elif resp.status_int == HTTP_UNAUTHORIZED: failed_files.append([wsgi_quote(str_to_wsgi(obj_name)), HTTPUnauthorized().status]) elif resp.status_int == HTTP_CONFLICT and pile and \ self.retry_count > 0 and self.retry_count > retry: retry += 1 sleep(self.retry_interval ** retry) delete_obj_req = Request.blank(resp.environ['PATH_INFO'], resp.environ) def _retry(req, app, obj_name, retry): return req.get_response(app), obj_name, retry pile.spawn(_retry, delete_obj_req, self.app, obj_name, retry) else: if resp.status_int // 100 == 5: failed_file_response['type'] = HTTPBadGateway failed_files.append([wsgi_quote(str_to_wsgi(obj_name)), resp.status])
def _get_source_object(self, req, path_info): # make a pre_auth request in case the user has write access # to container, but not READ. This was allowed in previous version # (i.e., before middleware) so keeping the same behavior here get_req = make_pre_authed_request( req.environ, path=wsgi_quote(path_info) + '?symlink=get', headers={'X-Newest': 'True'}, method='GET', swift_source='VW') source_resp = get_req.get_response(self.app) if source_resp.content_length is None or \ source_resp.content_length > MAX_FILE_SIZE: close_if_possible(source_resp.app_iter) return HTTPRequestEntityTooLarge(request=req) return source_resp
def _put_versioned_obj(self, req, put_path_info, source_resp): # Create a new Request object to PUT to the container, copying # all headers from the source object apart from x-timestamp. put_req = make_pre_authed_request( req.environ, path=wsgi_quote(put_path_info), method='PUT', swift_source='VW') copy_header_subset(source_resp, put_req, lambda k: k.lower() != 'x-timestamp') slo_size = put_req.headers.get('X-Object-Sysmeta-Slo-Size') if slo_size: put_req.headers['Content-Type'] += '; swift_bytes=' + slo_size put_req.environ['swift.content_type_overridden'] = True put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) put_resp = put_req.get_response(self.app) close_if_possible(source_resp.app_iter) return put_resp
def build_traversal_req(symlink_target): """ :returns: new request for target path if it's symlink otherwise None """ version, account, _junk = req.split_path(2, 3, True) account = self._response_header_value( TGT_ACCT_SYSMETA_SYMLINK_HDR) or wsgi_quote(account) target_path = os.path.join( '/', version, account, symlink_target.lstrip('/')) self._last_target_path = target_path new_req = make_subrequest( req.environ, path=target_path, method=req.method, headers=req.headers, swift_source='SYM') new_req.headers.pop('X-Backend-Storage-Policy-Index', None) return new_req
def handle_obj_versions_delete_push(self, req, versions_cont, api_version, account_name, container_name, object_name): """ Handle DELETE requests when in history mode. Copy current version of object to versions_container and write a delete marker before proceeding with original request. :param req: original request. :param versions_cont: container where previous versions of the object are stored. :param api_version: api version. :param account_name: account name. :param object_name: name of object of original request """ self._copy_current(req, versions_cont, api_version, account_name, object_name) marker_path = "/%s/%s/%s/%s" % ( api_version, account_name, versions_cont, self._build_versions_object_name(object_name, time.time())) marker_headers = { # Definitive source of truth is Content-Type, and since we add # a swift_* param, we know users haven't set it themselves. # This is still open to users POSTing to update the content-type # but they're just shooting themselves in the foot then. 'content-type': DELETE_MARKER_CONTENT_TYPE, 'content-length': '0', 'x-auth-token': req.headers.get('x-auth-token') } marker_req = make_pre_authed_request(req.environ, path=wsgi_quote(marker_path), headers=marker_headers, method='PUT', swift_source='VW') marker_req.environ['swift.content_type_overridden'] = True marker_resp = marker_req.get_response(self.app) self._check_response_error(req, marker_resp) drain_and_close(marker_resp) # successfully copied and created delete marker; safe to delete return self.app
def handle_obj_versions_delete_push(self, req, versions_cont, api_version, account_name, container_name, object_name): """ Handle DELETE requests when in history mode. Copy current version of object to versions_container and write a delete marker before proceeding with original request. :param req: original request. :param versions_cont: container where previous versions of the object are stored. :param api_version: api version. :param account_name: account name. :param object_name: name of object of original request """ self._copy_current(req, versions_cont, api_version, account_name, object_name) marker_path = "/%s/%s/%s/%s" % ( api_version, account_name, versions_cont, self._build_versions_object_name(object_name, time.time())) marker_headers = { # Definitive source of truth is Content-Type, and since we add # a swift_* param, we know users haven't set it themselves. # This is still open to users POSTing to update the content-type # but they're just shooting themselves in the foot then. 'content-type': DELETE_MARKER_CONTENT_TYPE, 'content-length': '0', 'x-auth-token': req.headers.get('x-auth-token')} marker_req = make_pre_authed_request( req.environ, path=wsgi_quote(marker_path), headers=marker_headers, method='PUT', swift_source='VW') marker_req.environ['swift.content_type_overridden'] = True marker_resp = marker_req.get_response(self.app) self._check_response_error(req, marker_resp) close_if_possible(marker_resp.app_iter) # successfully copied and created delete marker; safe to delete return self.app
def _list_versions(self, req, start_response, location): # Only supports JSON listings req.environ['swift.format_listing'] = False if not req.accept.best_match(['application/json']): raise HTTPNotAcceptable(request=req) params = req.params # FIXME(FVE) this is probably not working with oio-sds backend if 'version_marker' in params: if 'marker' not in params: raise HTTPBadRequest('version_marker param requires marker') if params['version_marker'] != 'null': try: ts = Timestamp(params.pop('version_marker')) except ValueError: raise HTTPBadRequest('invalid version_marker param') params['marker'] = self._build_versions_object_name( params['marker'], ts) delim = params.get('delimiter', '') # Exclude the set of chars used in version_id from user delimiters if set(delim).intersection('0123456789.%s' % RESERVED_STR): raise HTTPBadRequest('invalid delimiter param') null_listing = [] subdir_set = set() account = req.split_path(3, 3, True)[1] versions_req = make_pre_authed_request( req.environ, method='GET', swift_source='OV', path=wsgi_quote('/v1/%s/%s' % (account, location)), headers={'X-Backend-Allow-Reserved-Names': 'true'}, ) versions_req.environ['oio.query'] = {'versions': True} # NB: no end_marker support (yet) versions_req.params = { k: params.get(k, '') for k in ('prefix', 'marker', 'limit', 'delimiter', 'reverse', 'format') } versions_resp = versions_req.get_response(self.app) if versions_resp.status_int == HTTP_NOT_FOUND: raise versions_resp elif is_success(versions_resp.status_int): try: listing = json.loads(versions_resp.body) except ValueError: app_resp = [versions_resp.body] else: versions_listing = [] for item in listing: if 'subdir' in item: subdir_set.add(item['subdir']) else: item['version_id'] = str(item.get('version', 'null')) versions_listing.append(item) if (item['content_type'] == DELETE_MARKER_CONTENT_TYPE ): item['hash'] = MD5_OF_EMPTY_STRING subdir_listing = [{'subdir': s} for s in subdir_set] broken_listing = [] limit = constrain_req_limit(req, CONTAINER_LISTING_LIMIT) body = build_listing( null_listing, versions_listing, subdir_listing, broken_listing, reverse=config_true_value(params.get('reverse', 'no')), limit=limit, ) self.update_content_length(len(body)) app_resp = [body] else: return versions_resp(versions_req.environ, start_response) start_response(self._response_status, self._response_headers, self._response_exc_info) return app_resp
def test_check_symlink_header_invalid_format(self): def do_test(headers, status, err_msg): req = Request.blank('/v1/a/c/o', method='PUT', headers=headers) with self.assertRaises(swob.HTTPException) as cm: symlink._check_symlink_header(req) self.assertEqual(cm.exception.status, status) self.assertEqual(cm.exception.body, err_msg) do_test({'X-Symlink-Target': '/c1/o1'}, '412 Precondition Failed', b'X-Symlink-Target header must be of the ' b'form <container name>/<object name>') do_test({'X-Symlink-Target': 'c1o1'}, '412 Precondition Failed', b'X-Symlink-Target header must be of the ' b'form <container name>/<object name>') do_test( { 'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': '/another' }, '412 Precondition Failed', b'Account name cannot contain slashes') do_test( { 'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': 'an/other' }, '412 Precondition Failed', b'Account name cannot contain slashes') # url encoded case do_test({'X-Symlink-Target': '%2Fc1%2Fo1'}, '412 Precondition Failed', b'X-Symlink-Target header must be of the ' b'form <container name>/<object name>') do_test( { 'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': '%2Fanother' }, '412 Precondition Failed', b'Account name cannot contain slashes') do_test( { 'X-Symlink-Target': 'c1/o1', 'X-Symlink-Target-Account': 'an%2Fother' }, '412 Precondition Failed', b'Account name cannot contain slashes') # with multi-bytes do_test( { 'X-Symlink-Target': u'/\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' }, '412 Precondition Failed', b'X-Symlink-Target header must be of the ' b'form <container name>/<object name>') target = u'/\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' target = swob.bytes_to_wsgi(target.encode('utf8')) do_test({'X-Symlink-Target': swob.wsgi_quote(target)}, '412 Precondition Failed', b'X-Symlink-Target header must be of the ' b'form <container name>/<object name>') account = u'\u30b0\u30e9\u30d6\u30eb/\u30a2\u30ba\u30ec\u30f3' do_test( { 'X-Symlink-Target': 'c/o', 'X-Symlink-Target-Account': account }, '412 Precondition Failed', b'Account name cannot contain slashes') account = swob.bytes_to_wsgi(account.encode('utf8')) do_test( { 'X-Symlink-Target': 'c/o', 'X-Symlink-Target-Account': swob.wsgi_quote(account) }, '412 Precondition Failed', b'Account name cannot contain slashes')
def handle_obj_versions_delete_pop(self, req, versions_cont, api_version, account_name, container_name, object_name): """ Handle DELETE requests when in stack mode. Delete current version of object and pop previous version in its place. :param req: original request. :param versions_cont: container where previous versions of the object are stored. :param api_version: api version. :param account_name: account name. :param container_name: container name. :param object_name: object name. """ listing_prefix = self._build_versions_object_prefix(object_name) item_iter = self._listing_iter(account_name, versions_cont, listing_prefix, req) auth_token_header = {'X-Auth-Token': req.headers.get('X-Auth-Token')} authed = False for previous_version in item_iter: if not authed: # validate the write access to the versioned container before # making any backend requests if 'swift.authorize' in req.environ: container_info = get_container_info(req.environ, self.app, swift_source='VW') req.acl = container_info.get('write_acl') aresp = req.environ['swift.authorize'](req) if aresp: return aresp authed = True if previous_version['content_type'] == DELETE_MARKER_CONTENT_TYPE: # check whether we have data in the versioned container obj_head_headers = {'X-Newest': 'True'} obj_head_headers.update(auth_token_header) head_req = make_pre_authed_request(req.environ, path=wsgi_quote( req.path_info), method='HEAD', headers=obj_head_headers, swift_source='VW') hresp = head_req.get_response(self.app) drain_and_close(hresp) if hresp.status_int != HTTP_NOT_FOUND: self._check_response_error(req, hresp) # if there's an existing object, then just let the delete # through (i.e., restore to the delete-marker state): break # no data currently in the container (delete marker is current) for version_to_restore in item_iter: if version_to_restore['content_type'] == \ DELETE_MARKER_CONTENT_TYPE: # Nothing to restore break obj_to_restore = bytes_to_wsgi( version_to_restore['name'].encode('utf-8')) req.environ['QUERY_STRING'] = '' restored_path = self._restore_data( req, versions_cont, api_version, account_name, container_name, object_name, obj_to_restore) if not restored_path: continue old_del_req = make_pre_authed_request( req.environ, path=wsgi_quote(restored_path), method='DELETE', headers=auth_token_header, swift_source='VW') del_resp = old_del_req.get_response(self.app) drain_and_close(del_resp) if del_resp.status_int != HTTP_NOT_FOUND: self._check_response_error(req, del_resp) # else, well, it existed long enough to do the # copy; we won't worry too much break prev_obj_name = bytes_to_wsgi( previous_version['name'].encode('utf-8')) marker_path = "/%s/%s/%s/%s" % (api_version, account_name, versions_cont, prev_obj_name) # done restoring, redirect the delete to the marker req = make_pre_authed_request(req.environ, path=wsgi_quote(marker_path), method='DELETE', headers=auth_token_header, swift_source='VW') else: # there are older versions so copy the previous version to the # current object and delete the previous version prev_obj_name = bytes_to_wsgi( previous_version['name'].encode('utf-8')) req.environ['QUERY_STRING'] = '' restored_path = self._restore_data(req, versions_cont, api_version, account_name, container_name, object_name, prev_obj_name) if not restored_path: continue # redirect the original DELETE to the source of the reinstated # version object - we already auth'd original req so make a # pre-authed request req = make_pre_authed_request(req.environ, path=wsgi_quote(restored_path), method='DELETE', headers=auth_token_header, swift_source='VW') # remove 'X-If-Delete-At', since it is not for the older copy if 'X-If-Delete-At' in req.headers: del req.headers['X-If-Delete-At'] break # handle DELETE request here in case it was modified return req.get_response(self.app)
def _listing(self, env, start_response, prefix=None): """ Sends an HTML object listing to the remote client. :param env: The original WSGI environment dict. :param start_response: The original WSGI start_response hook. :param prefix: Any prefix desired for the container listing. """ label = wsgi_to_str(env['PATH_INFO']) if self._listings_label: groups = wsgi_to_str(env['PATH_INFO']).split('/') label = '{0}/{1}'.format(self._listings_label, '/'.join(groups[4:])) if not config_true_value(self._listings): body = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 ' \ 'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \ '<html>\n' \ '<head>\n' \ '<title>Listing of %s</title>\n' % cgi.escape(label) if self._listings_css: body += ' <link rel="stylesheet" type="text/css" ' \ 'href="%s" />\n' % self._build_css_path(prefix or '') else: body += ' <style type="text/css">\n' \ ' h1 {font-size: 1em; font-weight: bold;}\n' \ ' p {font-size: 2}\n' \ ' </style>\n' body += '</head>\n<body>' \ ' <h1>Web Listing Disabled</h1>' \ ' <p>The owner of this web site has disabled web listing.' \ ' <p>If you are the owner of this web site, you can enable' \ ' web listing by setting X-Container-Meta-Web-Listings.</p>' if self._index: body += '<h1>Index File Not Found</h1>' \ ' <p>The owner of this web site has set ' \ ' <b>X-Container-Meta-Web-Index: %s</b>. ' \ ' However, this file is not found.</p>' % self._index body += ' </body>\n</html>\n' resp = HTTPNotFound(body=body)(env, self._start_response) return self._error_response(resp, env, start_response) tmp_env = make_env( env, 'GET', '/%s/%s/%s' % ( self.version, self.account, self.container), self.agent, swift_source='SW') tmp_env['QUERY_STRING'] = 'delimiter=/' if prefix: tmp_env['QUERY_STRING'] += '&prefix=%s' % wsgi_quote(prefix) else: prefix = '' resp = self._app_call(tmp_env) if not is_success(self._get_status_int()): return self._error_response(resp, env, start_response) listing = None body = b''.join(resp) if body: listing = json.loads(body) if not listing: resp = HTTPNotFound()(env, self._start_response) return self._error_response(resp, env, start_response) headers = {'Content-Type': 'text/html; charset=UTF-8'} body = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 ' \ 'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \ '<html>\n' \ ' <head>\n' \ ' <title>Listing of %s</title>\n' % \ cgi.escape(label) if self._listings_css: body += ' <link rel="stylesheet" type="text/css" ' \ 'href="%s" />\n' % (self._build_css_path(prefix)) else: body += ' <style type="text/css">\n' \ ' h1 {font-size: 1em; font-weight: bold;}\n' \ ' th {text-align: left; padding: 0px 1em 0px 1em;}\n' \ ' td {padding: 0px 1em 0px 1em;}\n' \ ' a {text-decoration: none;}\n' \ ' </style>\n' body += ' </head>\n' \ ' <body>\n' \ ' <h1 id="title">Listing of %s</h1>\n' \ ' <table id="listing">\n' \ ' <tr id="heading">\n' \ ' <th class="colname">Name</th>\n' \ ' <th class="colsize">Size</th>\n' \ ' <th class="coldate">Date</th>\n' \ ' </tr>\n' % cgi.escape(label) if prefix: body += ' <tr id="parent" class="item">\n' \ ' <td class="colname"><a href="../">../</a></td>\n' \ ' <td class="colsize"> </td>\n' \ ' <td class="coldate"> </td>\n' \ ' </tr>\n' for item in listing: if 'subdir' in item: subdir = item['subdir'] if six.PY3 else \ item['subdir'].encode('utf-8') if prefix: subdir = subdir[len(prefix):] body += ' <tr class="item subdir">\n' \ ' <td class="colname"><a href="%s">%s</a></td>\n' \ ' <td class="colsize"> </td>\n' \ ' <td class="coldate"> </td>\n' \ ' </tr>\n' % \ (quote(subdir), cgi.escape(subdir)) for item in listing: if 'name' in item: name = item['name'] if six.PY3 else \ item['name'].encode('utf-8') if prefix: name = name[len(prefix):] content_type = item['content_type'] if six.PY3 else \ item['content_type'].encode('utf-8') bytes = human_readable(item['bytes']) last_modified = ( cgi.escape(item['last_modified'] if six.PY3 else item['last_modified'].encode('utf-8')). split('.')[0].replace('T', ' ')) body += ' <tr class="item %s">\n' \ ' <td class="colname"><a href="%s">%s</a></td>\n' \ ' <td class="colsize">%s</td>\n' \ ' <td class="coldate">%s</td>\n' \ ' </tr>\n' % \ (' '.join('type-' + cgi.escape(t.lower(), quote=True) for t in content_type.split('/')), quote(name), cgi.escape(name), bytes, last_modified) body += ' </table>\n' \ ' </body>\n' \ '</html>\n' resp = Response(headers=headers, body=body) return resp(env, start_response)
def get_source_resp(self, req): sub_req = make_subrequest(req.environ, path=wsgi_quote(req.path_info), headers=req.headers, swift_source='SSC') return sub_req.get_response(self.app)
def _listing(self, env, start_response, prefix=None): """ Sends an HTML object listing to the remote client. :param env: The original WSGI environment dict. :param start_response: The original WSGI start_response hook. :param prefix: Any WSGI-str prefix desired for the container listing. """ label = wsgi_to_str(env['PATH_INFO']) if self._listings_label: groups = wsgi_to_str(env['PATH_INFO']).split('/') label = '{0}/{1}'.format(self._listings_label, '/'.join(groups[4:])) if not config_true_value(self._listings): body = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 ' \ 'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \ '<html>\n' \ '<head>\n' \ '<title>Listing of %s</title>\n' % html_escape(label) if self._listings_css: body += ' <link rel="stylesheet" type="text/css" ' \ 'href="%s" />\n' % self._build_css_path(prefix or '') else: body += ' <style type="text/css">\n' \ ' h1 {font-size: 1em; font-weight: bold;}\n' \ ' p {font-size: 2}\n' \ ' </style>\n' body += '</head>\n<body>' \ ' <h1>Web Listing Disabled</h1>' \ ' <p>The owner of this web site has disabled web listing.' \ ' <p>If you are the owner of this web site, you can enable' \ ' web listing by setting X-Container-Meta-Web-Listings.</p>' if self._index: body += '<h1>Index File Not Found</h1>' \ ' <p>The owner of this web site has set ' \ ' <b>X-Container-Meta-Web-Index: %s</b>. ' \ ' However, this file is not found.</p>' % self._index body += ' </body>\n</html>\n' resp = HTTPNotFound(body=body)(env, self._start_response) return self._error_response(resp, env, start_response) tmp_env = make_env(env, 'GET', '/%s/%s/%s' % (self.version, self.account, self.container), self.agent, swift_source='SW') tmp_env['QUERY_STRING'] = 'delimiter=/' if prefix: tmp_env['QUERY_STRING'] += '&prefix=%s' % wsgi_quote(prefix) else: prefix = '' resp = self._app_call(tmp_env) if not is_success(self._get_status_int()): return self._error_response(resp, env, start_response) listing = None body = b''.join(resp) if body: listing = json.loads(body) if not listing: resp = HTTPNotFound()(env, self._start_response) return self._error_response(resp, env, start_response) headers = {'Content-Type': 'text/html; charset=UTF-8'} body = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 ' \ 'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \ '<html>\n' \ ' <head>\n' \ ' <title>Listing of %s</title>\n' % \ html_escape(label) if self._listings_css: body += ' <link rel="stylesheet" type="text/css" ' \ 'href="%s" />\n' % (self._build_css_path(prefix)) else: body += ' <style type="text/css">\n' \ ' h1 {font-size: 1em; font-weight: bold;}\n' \ ' th {text-align: left; padding: 0px 1em 0px 1em;}\n' \ ' td {padding: 0px 1em 0px 1em;}\n' \ ' a {text-decoration: none;}\n' \ ' </style>\n' body += ' </head>\n' \ ' <body>\n' \ ' <h1 id="title">Listing of %s</h1>\n' \ ' <table id="listing">\n' \ ' <tr id="heading">\n' \ ' <th class="colname">Name</th>\n' \ ' <th class="colsize">Size</th>\n' \ ' <th class="coldate">Date</th>\n' \ ' </tr>\n' % html_escape(label) if prefix: body += ' <tr id="parent" class="item">\n' \ ' <td class="colname"><a href="../">../</a></td>\n' \ ' <td class="colsize"> </td>\n' \ ' <td class="coldate"> </td>\n' \ ' </tr>\n' for item in listing: if 'subdir' in item: subdir = item['subdir'] if six.PY3 else \ item['subdir'].encode('utf-8') if prefix: subdir = subdir[len(wsgi_to_str(prefix)):] body += ' <tr class="item subdir">\n' \ ' <td class="colname"><a href="%s">%s</a></td>\n' \ ' <td class="colsize"> </td>\n' \ ' <td class="coldate"> </td>\n' \ ' </tr>\n' % \ (quote(subdir), html_escape(subdir)) for item in listing: if 'name' in item: name = item['name'] if six.PY3 else \ item['name'].encode('utf-8') if prefix: name = name[len(wsgi_to_str(prefix)):] content_type = item['content_type'] if six.PY3 else \ item['content_type'].encode('utf-8') bytes = human_readable(item['bytes']) last_modified = (html_escape( item['last_modified'] if six.PY3 else item['last_modified'] .encode('utf-8')).split('.')[0].replace('T', ' ')) body += ' <tr class="item %s">\n' \ ' <td class="colname"><a href="%s">%s</a></td>\n' \ ' <td class="colsize">%s</td>\n' \ ' <td class="coldate">%s</td>\n' \ ' </tr>\n' % \ (' '.join('type-' + html_escape(t.lower()) for t in content_type.split('/')), quote(name), html_escape(name), bytes, last_modified) body += ' </table>\n' \ ' </body>\n' \ '</html>\n' resp = Response(headers=headers, body=body) return resp(env, start_response)
def handle_obj_versions_delete_pop(self, req, versions_cont, api_version, account_name, container_name, object_name): """ Handle DELETE requests when in stack mode. Delete current version of object and pop previous version in its place. :param req: original request. :param versions_cont: container where previous versions of the object are stored. :param api_version: api version. :param account_name: account name. :param container_name: container name. :param object_name: object name. """ listing_prefix = self._build_versions_object_prefix(object_name) item_iter = self._listing_iter(account_name, versions_cont, listing_prefix, req) auth_token_header = {'X-Auth-Token': req.headers.get('X-Auth-Token')} authed = False for previous_version in item_iter: if not authed: # validate the write access to the versioned container before # making any backend requests if 'swift.authorize' in req.environ: container_info = get_container_info( req.environ, self.app) req.acl = container_info.get('write_acl') aresp = req.environ['swift.authorize'](req) if aresp: return aresp authed = True if previous_version['content_type'] == DELETE_MARKER_CONTENT_TYPE: # check whether we have data in the versioned container obj_head_headers = {'X-Newest': 'True'} obj_head_headers.update(auth_token_header) head_req = make_pre_authed_request( req.environ, path=wsgi_quote(req.path_info), method='HEAD', headers=obj_head_headers, swift_source='VW') hresp = head_req.get_response(self.app) close_if_possible(hresp.app_iter) if hresp.status_int != HTTP_NOT_FOUND: self._check_response_error(req, hresp) # if there's an existing object, then just let the delete # through (i.e., restore to the delete-marker state): break # no data currently in the container (delete marker is current) for version_to_restore in item_iter: if version_to_restore['content_type'] == \ DELETE_MARKER_CONTENT_TYPE: # Nothing to restore break obj_to_restore = bytes_to_wsgi( version_to_restore['name'].encode('utf-8')) restored_path = self._restore_data( req, versions_cont, api_version, account_name, container_name, object_name, obj_to_restore) if not restored_path: continue old_del_req = make_pre_authed_request( req.environ, path=wsgi_quote(restored_path), method='DELETE', headers=auth_token_header, swift_source='VW') del_resp = old_del_req.get_response(self.app) close_if_possible(del_resp.app_iter) if del_resp.status_int != HTTP_NOT_FOUND: self._check_response_error(req, del_resp) # else, well, it existed long enough to do the # copy; we won't worry too much break prev_obj_name = bytes_to_wsgi( previous_version['name'].encode('utf-8')) marker_path = "/%s/%s/%s/%s" % ( api_version, account_name, versions_cont, prev_obj_name) # done restoring, redirect the delete to the marker req = make_pre_authed_request( req.environ, path=wsgi_quote(marker_path), method='DELETE', headers=auth_token_header, swift_source='VW') else: # there are older versions so copy the previous version to the # current object and delete the previous version prev_obj_name = bytes_to_wsgi( previous_version['name'].encode('utf-8')) restored_path = self._restore_data( req, versions_cont, api_version, account_name, container_name, object_name, prev_obj_name) if not restored_path: continue # redirect the original DELETE to the source of the reinstated # version object - we already auth'd original req so make a # pre-authed request req = make_pre_authed_request( req.environ, path=wsgi_quote(restored_path), method='DELETE', headers=auth_token_header, swift_source='VW') # remove 'X-If-Delete-At', since it is not for the older copy if 'X-If-Delete-At' in req.headers: del req.headers['X-If-Delete-At'] break # handle DELETE request here in case it was modified return req.get_response(self.app)
def handle_extract_iter(self, req, compress_type, out_content_type='text/plain'): """ A generator that can be assigned to a swob Response's app_iter which, when iterated over, will extract and PUT the objects pulled from the request body. Will occasionally yield whitespace while request is being processed. When the request is completed will yield a response body that can be parsed to determine success. See above documentation for details. :params req: a swob Request :params compress_type: specifying the compression type of the tar. Accepts '', 'gz', or 'bz2' """ resp_dict = { 'Response Status': HTTPCreated().status, 'Response Body': '', 'Number Files Created': 0 } failed_files = [] last_yield = time() if out_content_type and out_content_type.endswith('/xml'): to_yield = b'<?xml version="1.0" encoding="UTF-8"?>\n' else: to_yield = b' ' separator = b'' containers_accessed = set() req.environ['eventlet.minimum_write_chunk_size'] = 0 try: if not out_content_type: raise HTTPNotAcceptable(request=req) if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) try: vrs, account, extract_base = req.split_path(2, 3, True) except ValueError: raise HTTPNotFound(request=req) extract_base = extract_base or '' extract_base = extract_base.rstrip('/') tar = tarfile.open(mode='r|' + compress_type, fileobj=req.body_file) failed_response_type = HTTPBadRequest containers_created = 0 while True: if last_yield + self.yield_frequency < time(): last_yield = time() yield to_yield to_yield, separator = b' ', b'\r\n\r\n' tar_info = tar.next() if tar_info is None or \ len(failed_files) >= self.max_failed_extractions: break if tar_info.isfile(): obj_path = tar_info.name if not six.PY2: obj_path = obj_path.encode('utf-8', 'surrogateescape') obj_path = bytes_to_wsgi(obj_path) if obj_path.startswith('./'): obj_path = obj_path[2:] obj_path = obj_path.lstrip('/') if extract_base: obj_path = extract_base + '/' + obj_path if '/' not in obj_path: continue # ignore base level file destination = '/'.join(['', vrs, account, obj_path]) container = obj_path.split('/', 1)[0] if not constraints.check_utf8(wsgi_to_str(destination)): failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPPreconditionFailed().status ]) continue if tar_info.size > constraints.MAX_FILE_SIZE: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPRequestEntityTooLarge().status ]) continue container_failure = None if container not in containers_accessed: cont_path = '/'.join(['', vrs, account, container]) try: if self.create_container(req, cont_path): containers_created += 1 if containers_created > self.max_containers: raise HTTPBadRequest( 'More than %d containers to create ' 'from tar.' % self.max_containers) except CreateContainerError as err: # the object PUT to this container still may # succeed if acls are set container_failure = [ wsgi_quote(cont_path[:self.max_path_length]), err.status ] if err.status_int == HTTP_UNAUTHORIZED: raise HTTPUnauthorized(request=req) except ValueError: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPBadRequest().status ]) continue tar_file = tar.extractfile(tar_info) create_headers = { 'Content-Length': tar_info.size, 'X-Auth-Token': req.headers.get('X-Auth-Token'), } # Copy some whitelisted headers to the subrequest for k, v in req.headers.items(): if ((k.lower() in ('x-delete-at', 'x-delete-after')) or is_user_meta('object', k)): create_headers[k] = v create_obj_req = make_subrequest( req.environ, method='PUT', path=wsgi_quote(destination), headers=create_headers, agent='%(orig)s BulkExpand', swift_source='EA') create_obj_req.environ['wsgi.input'] = tar_file for pax_key, pax_value in tar_info.pax_headers.items(): header_name = pax_key_to_swift_header(pax_key) if header_name: # Both pax_key and pax_value are unicode # strings; the key is already UTF-8 encoded, but # we still have to encode the value. create_obj_req.headers[header_name] = \ pax_value.encode("utf-8") resp = create_obj_req.get_response(self.app) containers_accessed.add(container) if resp.is_success: resp_dict['Number Files Created'] += 1 else: if container_failure: failed_files.append(container_failure) if resp.status_int == HTTP_UNAUTHORIZED: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPUnauthorized().status ]) raise HTTPUnauthorized(request=req) if resp.status_int // 100 == 5: failed_response_type = HTTPBadGateway failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), resp.status ]) if failed_files: resp_dict['Response Status'] = failed_response_type().status elif not resp_dict['Number Files Created']: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files' except HTTPException as err: resp_dict['Response Status'] = err.status resp_dict['Response Body'] = err.body.decode('utf-8') except (tarfile.TarError, zlib.error) as tar_error: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error except Exception: self.logger.exception('Error in extract archive.') resp_dict['Response Status'] = HTTPServerError().status yield separator + get_response_body(out_content_type, resp_dict, failed_files, 'extract')
def get_source_resp(self, req): sub_req = make_subrequest( req.environ, path=wsgi_quote(req.path_info), headers=req.headers, swift_source='SSC') return sub_req.get_response(self.app)
def handle_extract_iter(self, req, compress_type, out_content_type='text/plain'): """ A generator that can be assigned to a swob Response's app_iter which, when iterated over, will extract and PUT the objects pulled from the request body. Will occasionally yield whitespace while request is being processed. When the request is completed will yield a response body that can be parsed to determine success. See above documentation for details. :params req: a swob Request :params compress_type: specifying the compression type of the tar. Accepts '', 'gz', or 'bz2' """ resp_dict = {'Response Status': HTTPCreated().status, 'Response Body': '', 'Number Files Created': 0} failed_files = [] last_yield = time() if out_content_type and out_content_type.endswith('/xml'): to_yield = b'<?xml version="1.0" encoding="UTF-8"?>\n' else: to_yield = b' ' separator = b'' containers_accessed = set() req.environ['eventlet.minimum_write_chunk_size'] = 0 try: if not out_content_type: raise HTTPNotAcceptable(request=req) if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) try: vrs, account, extract_base = req.split_path(2, 3, True) except ValueError: raise HTTPNotFound(request=req) extract_base = extract_base or '' extract_base = extract_base.rstrip('/') tar = tarfile.open(mode='r|' + compress_type, fileobj=req.body_file) failed_response_type = HTTPBadRequest containers_created = 0 while True: if last_yield + self.yield_frequency < time(): last_yield = time() yield to_yield to_yield, separator = b' ', b'\r\n\r\n' tar_info = tar.next() if tar_info is None or \ len(failed_files) >= self.max_failed_extractions: break if tar_info.isfile(): obj_path = tar_info.name if not six.PY2: obj_path = obj_path.encode('utf-8', 'surrogateescape') obj_path = bytes_to_wsgi(obj_path) if obj_path.startswith('./'): obj_path = obj_path[2:] obj_path = obj_path.lstrip('/') if extract_base: obj_path = extract_base + '/' + obj_path if '/' not in obj_path: continue # ignore base level file destination = '/'.join( ['', vrs, account, obj_path]) container = obj_path.split('/', 1)[0] if not constraints.check_utf8(wsgi_to_str(destination)): failed_files.append( [wsgi_quote(obj_path[:self.max_path_length]), HTTPPreconditionFailed().status]) continue if tar_info.size > constraints.MAX_FILE_SIZE: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPRequestEntityTooLarge().status]) continue container_failure = None if container not in containers_accessed: cont_path = '/'.join(['', vrs, account, container]) try: if self.create_container(req, cont_path): containers_created += 1 if containers_created > self.max_containers: raise HTTPBadRequest( 'More than %d containers to create ' 'from tar.' % self.max_containers) except CreateContainerError as err: # the object PUT to this container still may # succeed if acls are set container_failure = [ wsgi_quote(cont_path[:self.max_path_length]), err.status] if err.status_int == HTTP_UNAUTHORIZED: raise HTTPUnauthorized(request=req) except ValueError: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPBadRequest().status]) continue tar_file = tar.extractfile(tar_info) create_headers = { 'Content-Length': tar_info.size, 'X-Auth-Token': req.headers.get('X-Auth-Token'), } create_obj_req = make_subrequest( req.environ, method='PUT', path=wsgi_quote(destination), headers=create_headers, agent='%(orig)s BulkExpand', swift_source='EA') create_obj_req.environ['wsgi.input'] = tar_file for pax_key, pax_value in tar_info.pax_headers.items(): header_name = pax_key_to_swift_header(pax_key) if header_name: # Both pax_key and pax_value are unicode # strings; the key is already UTF-8 encoded, but # we still have to encode the value. create_obj_req.headers[header_name] = \ pax_value.encode("utf-8") resp = create_obj_req.get_response(self.app) containers_accessed.add(container) if resp.is_success: resp_dict['Number Files Created'] += 1 else: if container_failure: failed_files.append(container_failure) if resp.status_int == HTTP_UNAUTHORIZED: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPUnauthorized().status]) raise HTTPUnauthorized(request=req) if resp.status_int // 100 == 5: failed_response_type = HTTPBadGateway failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), resp.status]) if failed_files: resp_dict['Response Status'] = failed_response_type().status elif not resp_dict['Number Files Created']: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files' except HTTPException as err: resp_dict['Response Status'] = err.status resp_dict['Response Body'] = err.body.decode('utf-8') except (tarfile.TarError, zlib.error) as tar_error: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error except Exception: self.logger.exception('Error in extract archive.') resp_dict['Response Status'] = HTTPServerError().status yield separator + get_response_body( out_content_type, resp_dict, failed_files, 'extract')