def handle_request(self, env, req): """ Handles the POST /origin/.prep call for preparing the backing store Swift cluster for use with the origin subsystem. Can only be called by .origin_admin :param req: The webob.Request to process. :returns: webob.Response, 204 on success """ if not self.is_origin_admin(req): return HTTPForbidden(request=req) try: vsn, account = split_path(req.path, 2, 2) except ValueError: return HTTPBadRequest(request=req) if account == '.prep': path = '/v1/%s' % self.origin_account resp = make_pre_authed_request(req.environ, 'PUT', path, agent='SwiftOrigin').get_response(self.app) if resp.status_int // 100 != 2: raise Exception( 'Could not create the main origin account: %s %s' % (path, resp.status)) for i in xrange(self.num_hash_cont): cont_name = '.hash_%d' % i path = '/v1/%s/%s' % (self.origin_account, cont_name) resp = make_pre_authed_request(req.environ, 'PUT', path, agent='SwiftOrigin').get_response(self.app) if resp.status_int // 100 != 2: raise Exception('Could not create %s container: %s %s' % (cont_name, path, resp.status)) return HTTPNoContent(request=req) return HTTPNotFound(request=req)
def test_pre_auth_req_drops_query(self): r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path") self.assertEquals(r.query_string, "original") r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path?replacement") self.assertEquals(r.query_string, "replacement") r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path?") self.assertEquals(r.query_string, "")
def _set_hash_data(self, env, cdn_obj_path, new_hash_data, update_listings=True): """ Actually sets the data in the .origin account. If not successful on any of the several updates this has to do, will raise a OriginDbFailure """ cdn_obj_data = new_hash_data.get_json_str() cdn_obj_etag = md5(cdn_obj_data).hexdigest() # this is always a PUT because a POST needs to update the file cdn_obj_resp = make_pre_authed_request( env, 'PUT', cdn_obj_path, body=cdn_obj_data, headers={'Etag': cdn_obj_etag}, agent='SwiftOrigin', swift_source='SOS').get_response(self.app) if cdn_obj_resp.status_int // 100 != 2: raise OriginDbFailure( 'Could not PUT .hash obj in origin ' 'db: %s %s' % (cdn_obj_path, cdn_obj_resp.status_int)) memcache_client = utils.cache_from_env(env) if memcache_client: memcache_key = self.cdn_data_memcache_key(cdn_obj_path) memcache_client.delete(memcache_key) if not update_listings: return listing_cont_path = quote('/v1/%s/%s' % (self.origin_account, new_hash_data.account)) resp = make_pre_authed_request( env, 'HEAD', listing_cont_path, agent='SwiftOrigin', swift_source='SOS').get_response(self.app) if resp.status_int == 404: # create new container for listings resp = make_pre_authed_request( env, 'PUT', listing_cont_path, agent='SwiftOrigin', swift_source='SOS').get_response(self.app) if resp.status_int // 100 != 2: raise OriginDbFailure( 'Could not create listing container ' 'in origin db: %s %s' % (listing_cont_path, resp.status)) cdn_list_path = quote('/v1/%s/%s/%s' % ( self.origin_account, new_hash_data.account.encode('utf-8'), new_hash_data.container.encode('utf-8'))) listing_content_type = new_hash_data.gen_listing_content_type() cdn_list_resp = make_pre_authed_request( env, 'PUT', cdn_list_path, headers={'Content-Type': listing_content_type, 'Content-Length': 0}, agent='SwiftOrigin', swift_source='SOS').get_response(self.app) if cdn_list_resp.status_int // 100 != 2: raise OriginDbFailure( 'Could not PUT/POST to cdn listing in ' 'origin db: %s %s' % (cdn_list_path, cdn_list_resp.status_int))
def test_pre_auth_req_drops_query(self): r = wsgi.make_pre_authed_request( {'QUERY_STRING': 'original'}, 'GET', 'path') self.assertEquals(r.query_string, 'original') r = wsgi.make_pre_authed_request( {'QUERY_STRING': 'original'}, 'GET', 'path?replacement') self.assertEquals(r.query_string, 'replacement') r = wsgi.make_pre_authed_request( {'QUERY_STRING': 'original'}, 'GET', 'path?') self.assertEquals(r.query_string, '')
def test_pre_auth_req(self): class FakeReq(object): @classmethod def fake_blank(cls, path, environ={}, body="", headers={}): self.assertEquals(environ["swift.authorize"]("test"), None) self.assertFalse("HTTP_X_TRANS_ID" in environ) was_blank = Request.blank Request.blank = FakeReq.fake_blank wsgi.make_pre_authed_request({"HTTP_X_TRANS_ID": "1234"}, "PUT", "/", body="tester", headers={}) wsgi.make_pre_authed_request({"HTTP_X_TRANS_ID": "1234"}, "PUT", "/", headers={}) Request.blank = was_blank
def test_pre_auth_req(self): class FakeReq(object): @classmethod def fake_blank(cls, path, environ={}, body='', headers={}): self.assertEquals(environ['swift.authorize']('test'), None) self.assertFalse('HTTP_X_TRANS_ID' in environ) was_blank = Request.blank Request.blank = FakeReq.fake_blank wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'}, 'PUT', '/', body='tester', headers={}) wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'}, 'PUT', '/', headers={}) Request.blank = was_blank
def test_pre_auth_req_with_env_path_and_script(self): env = {"PATH_INFO": "/unquoted path with %20", "SCRIPT_NAME": "/script"} r = wsgi.make_pre_authed_request(env, "GET") expected_path = quote(env["SCRIPT_NAME"] + env["PATH_INFO"]) self.assertEquals(r.path, expected_path) env = {"PATH_INFO": "", "SCRIPT_NAME": "/script"} r = wsgi.make_pre_authed_request(env, "GET") self.assertEquals(r.path, "/script") env = {"PATH_INFO": "/path", "SCRIPT_NAME": ""} r = wsgi.make_pre_authed_request(env, "GET") self.assertEquals(r.path, "/path") env = {"PATH_INFO": "", "SCRIPT_NAME": ""} r = wsgi.make_pre_authed_request(env, "GET") self.assertEquals(r.path, "")
def origin_db_delete(self, env, req): """ Handles DELETEs in the Origin database """ if not self.delete_enabled: return HTTPMethodNotAllowed(request=req) try: vsn, account, container = split_path(req.path, 3, 3) except ValueError: return HTTPBadRequest( 'Invalid request. ' 'URI format: /<api version>/<account>/<container>') hsh = self.hash_path(account, container) cdn_obj_path = self.get_hsh_obj_path(hsh) # Remove memcache entry memcache_client = utils.cache_from_env(env) if memcache_client: memcache_key = self.cdn_data_memcache_key(cdn_obj_path) memcache_client.delete(memcache_key) resp = make_pre_authed_request(env, 'DELETE', cdn_obj_path, agent='SwiftOrigin', swift_source='SOS').get_response( self.app) # A 404 means it's already deleted, which is okay if resp.status_int // 100 != 2 and resp.status_int != 404: raise OriginDbFailure('Could not DELETE .hash obj in origin ' 'db: %s %s' % (cdn_obj_path, resp.status_int)) cdn_list_path = quote('/v1/%s/%s/%s' % (self.origin_account, account, container)) list_resp = make_pre_authed_request(env, 'DELETE', cdn_list_path, agent='SwiftOrigin', swift_source='SOS').get_response( self.app) if list_resp.status_int // 100 != 2 and list_resp.status_int != 404: raise OriginDbFailure('Could not DELETE listing path in origin ' 'db: %s %s' % (cdn_list_path, list_resp.status_int)) # Return 404 if container didn't exist if resp.status_int == 404 and list_resp.status_int == 404: return HTTPNotFound(request=req) return HTTPNoContent(request=req)
def test_pre_auth_req_with_env_path_and_script(self): env = {'PATH_INFO': '/unquoted path with %20', 'SCRIPT_NAME': '/script'} r = wsgi.make_pre_authed_request(env, 'GET') expected_path = quote(env['SCRIPT_NAME'] + env['PATH_INFO']) self.assertEquals(r.path, expected_path) env = {'PATH_INFO': '', 'SCRIPT_NAME': '/script'} r = wsgi.make_pre_authed_request(env, 'GET') self.assertEquals(r.path, '/script') env = {'PATH_INFO': '/path', 'SCRIPT_NAME': ''} r = wsgi.make_pre_authed_request(env, 'GET') self.assertEquals(r.path, '/path') env = {'PATH_INFO': '', 'SCRIPT_NAME': ''} r = wsgi.make_pre_authed_request(env, 'GET') self.assertEquals(r.path, '')
def _get_details(self, req, access_key): """Get access key details. :return: (secret_key, account) as tuple or (None, None) if not found. """ memcache_client = cache_from_env(req.environ) if memcache_client: memcache_key = MEMCACHE_KEY_FORMAT % (self.reseller_prefix, access_key) data = memcache_client.get(memcache_key) if data: return data[0], data[1] path = quote(self.akd_container_url + access_key) resp = make_pre_authed_request(req.environ, 'GET', path).get_response(self.app) if resp.status_int // 100 == 2: data = json.loads(resp.body) secret_key, account = data['secret_key'], data['account'] if memcache_client: memcache_client.set(memcache_key, (secret_key, account), time=self.cache_time) return secret_key, account elif resp.status_int // 100 == 4: return None, None else: raise Exception('Could not GET access key details: {} {}'.format( path, resp.status_int))
def get_listings(marker): listing_path = quote('/v1/%s/%s' % (self.origin_account, account)) listing_path += '?format=json&marker=' + quote(marker) # no limit in request because may have to filter on cdn_enabled resp = make_pre_authed_request( env, 'GET', listing_path, agent='SwiftOrigin', swift_source='SOS').get_response(self.app) resp_headers = {} listing_formatted = [] if resp.status_int // 100 == 2: cont_listing = json.loads(resp.body) for listing_dict in cont_listing: if limit is None or len(listing_formatted) < limit: try: formatted_data = self._parse_container_listing( account, listing_dict, list_format, only_cdn_enabled=enabled_only) if formatted_data: listing_formatted.append(formatted_data) except InvalidContentType, e: self.logger.exception(e) continue else: break if cont_listing and not listing_formatted: # there were rows returned but none matched enabled_only- # requery with new marker new_marker = cont_listing[-1]['name'] if isinstance(new_marker, unicode): new_marker = new_marker.encode('utf-8') return get_listings(new_marker)
def iter_objects_by_prefix(account, container, prefix, swift_client=None, app=None): marker = '' while True: param = 'format=json&marker=%s' % marker if marker == '': param = '%s&prefix=%s' % (param, prefix) if swift_client: path = swift_client.make_path(account, container) resp = swift_client.make_request('GET', '%s?%s' % (path, param), {}, (2, 4)) elif app: path = '/v1/%s/%s' % (account, container) env = make_pre_authed_env({}, method='GET', path=path, query_string=param) req = make_pre_authed_request(env) resp = req.get_response(app) if not resp.status_int == 200: break data = json.loads(resp.body) if not data: break for item in data: yield item marker = data[-1]['name'].encode('utf8')
def get_account_info(env, app): """ Get the info structure for an account, based on env and app. This is useful to middlewares. Note: This call bypasses auth. Success does not imply that the request has authorization to the account_info. """ (version, account, container, _) = \ split_path(env['PATH_INFO'], 2, 4, True) new_env = env.copy() obj_path = '/%s/%s' % (version, account) if isinstance(obj_path, unicode): obj_path = obj_path.encode('utf-8') new_env['PATH_INFO'] = obj_path if new_env.has_key('wsgi.input'): del(new_env['wsgi.input']) if new_env.has_key('QUERY_STRING'): del(new_env['QUERY_STRING']) new_env['CONTENT_LENGTH'] = 0 resp = make_pre_authed_request(new_env, 'HEAD', '/%s/%s' % (version, account)).get_response(app) account_info = headers_to_account_info( resp.headers, resp.status_int) return account_info
def _put_versioned_obj(self, req, put_path_info, source_resp): # Create a new Request object to PUT to the versions container, copying # all headers from the source object apart from x-timestamp. put_req = make_pre_authed_request(req.environ, path=put_path_info, method="PUT", swift_source="VW") copy_header_subset(source_resp, put_req, lambda k: k.lower() != "x-timestamp") put_req.environ["wsgi.input"] = FileLikeIter(source_resp.app_iter) return put_req.get_response(self.app)
def get_chunks(self, env, chunks, container): file_compress_content = [] self.app.logger.info( 'StackSync API: get_chunks: chunks: %s container: %s', str(chunks), str(container)) seg_resp = None for chunk in chunks: new_path = "/v1/" + env[ 'stacksync_user_account'] + "/" + container + "/" + str(chunk) seg_req = make_pre_authed_request(env, method='GET', path=new_path, body="", agent='%(orig)s') seg_resp = seg_req.get_response(self.app) if is_valid_status(seg_resp.status_int): file_compress_content.append(seg_resp.body) else: file_compress_content = [] break if seg_resp: return file_compress_content, seg_resp.status_int else: # No chunks return file_compress_content, 200
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env): marker = '' while True: lreq = make_pre_authed_request(env, method='GET', swift_source='VW', path='/v1/%s/%s' % (account_name, lcontainer)) lreq.environ['QUERY_STRING'] = \ 'format=json&prefix=%s&reverse=on&marker=%s' % ( quote(lprefix), quote(marker)) lresp = lreq.get_response(self.app) if not is_success(lresp.status_int): if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif is_client_error(lresp.status_int): raise HTTPPreconditionFailed() else: raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break marker = sublisting[-1]['name'].encode('utf-8') yield sublisting
def _get_container_info(self, env): """ Retrieves x-container-meta-web-index, x-container-meta-web-error, x-container-meta-web-listings, and x-container-meta-web-listings-css from memcache or from the cluster and stores the result in memcache and in self._index, self._error, self._listings, and self._listings_css. :param env: The WSGI environment dict. """ self._index = self._error = self._listings = self._listings_css = None memcache_client = cache_from_env(env) if memcache_client: memcache_key = "/staticweb/%s/%s/%s" % (self.version, self.account, self.container) cached_data = memcache_client.get(memcache_key) if cached_data: (self._index, self._error, self._listings, self._listings_css) = cached_data return resp = make_pre_authed_request( env, "HEAD", "/%s/%s/%s" % (self.version, self.account, self.container), agent=self.agent ).get_response(self.app) if is_success(resp.status_int): self._index = resp.headers.get("x-container-meta-web-index", "").strip() self._error = resp.headers.get("x-container-meta-web-error", "").strip() self._listings = resp.headers.get("x-container-meta-web-listings", "").strip() self._listings_css = resp.headers.get("x-container-meta-web-listings-css", "").strip() if memcache_client: memcache_client.set( memcache_key, (self._index, self._error, self._listings, self._listings_css), timeout=self.cache_timeout, )
def iter_objects(self, env, path, prefix, marker, end, count): path_with_params = '%s?format=json&prefix=%s' % (path, prefix) seg = '' force_break = False while count > 0: l = 1000 if count > 1000 else count count -= 1000 rpath = path_with_params + ('&marker=%s' % marker) + ( '&limit=%d' % l) req = make_pre_authed_request(env, 'GET', rpath) req.environ['swift.proxy_access_log_made'] = True resp = req.get_response(self.app) segments = json.loads(resp.body) for seg in segments: name = seg['name'] record_ts = int(name.split('/')[1]) if record_ts > end: force_break = True break yield name if force_break: break if len(segments) != l: break if segments: marker = seg['name'] else: break
def origin_db_get(self, env, req): ''' Handles GETs to the Origin database The only part of the path this pays attention to is the account. ''' #TODO: this does not return transfer-encoding: chunked try: account = req.path.split('/')[2] except IndexError: return HTTPBadRequest('Invalid request. ' 'URI format: /<api version>/<account>') #TODO: make sure to test with unicode container names marker = get_param(req, 'marker', default='') list_format = get_param(req, 'format') enabled_only = get_param(req, 'enabled', default='false') in TRUE_VALUES limit = get_param(req, 'limit') if limit: try: limit = int(limit) except ValueError: return HTTPBadRequest('Invalid limit, must be an integer') listing_path = '/v1/%s/%s?format=json&marker=%s' % \ (self.origin_account, account, marker) # no limit in request because may have to filter on cdn_enabled resp = make_pre_authed_request(env, 'GET', listing_path, agent='SwiftOrigin').get_response(self.app) resp_headers = {} # {'Transfer-Encoding': 'chunked'} #TODO is this right? was chunked in old one if resp.status_int // 100 == 2: cont_listing = json.loads(resp.body) # TODO: is it ok to load the whole thing? do i have a choice? listing_formatted = [] for listing_dict in cont_listing: if limit is None or len(listing_formatted) < limit: try: formatted_data = self._parse_container_listing( account, listing_dict, list_format, only_cdn_enabled=enabled_only) if formatted_data: listing_formatted.append(formatted_data) except InvalidContentType, e: self.logger.exception(e) continue else: break if list_format == 'xml': resp_headers['Content-Type'] = 'application/xml' response_body = ('<?xml version="1.0" encoding="UTF-8"?>\n' '<account name="%s">\n%s\n</account>') % (account, '\n'.join(listing_formatted)) elif list_format == 'json': resp_headers['Content-Type'] = 'application/json' response_body = json.dumps(listing_formatted) else: resp_headers['Content-Type'] = 'text/plain; charset=UTF-8' response_body = '\n'.join(listing_formatted) return Response(body=response_body, headers=resp_headers)
def remove_old_chunks(self, env, chunks_diff, container): error = False self.app.logger.info('StackSync API: remove old chunks: container: %s', str(container)) for chunk_name in chunks_diff: env_aux = env.copy() new_path = "/v1/" + env[ 'stacksync_user_account'] + "/" + container + "/" + str( chunk_name) del env_aux['HTTP_STACKSYNC_API'] seg_req = make_pre_authed_request(env_aux, method='DELETE', path=new_path, agent=str(container)) seg_resp = seg_req.get_response(self.app) if not is_valid_status(seg_resp.status_int): self.app.logger.error( 'StackSync API: upload_file_chunks: error deleting old chunk %s', str(chunk_name)) error = True break if error: self.app.logger.error( 'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend', seg_resp.status) response = create_error_response( 500, "Error uploading chunks to storage backend") else: response = HTTPCreated() return response
def get_container_info(env, app, swift_source=None): """ Get the info structure for a container, based on env and app. This is useful to middlewares. """ cache = cache_from_env(env) if not cache: return None (version, account, container, obj) = \ split_path(env['PATH_INFO'], 2, 4, True) cache_key = get_container_memcache_key(account, container) # Use a unique environment cache key per container. If you copy this env # to make a new request, it won't accidentally reuse the old container info env_key = 'swift.%s' % cache_key if env_key not in env: container_info = cache.get(cache_key) if not container_info: resp = make_pre_authed_request( env, 'HEAD', '/%s/%s/%s' % (version, account, container), swift_source=swift_source, ).get_response(app) container_info = headers_to_container_info( resp.headers, resp.status_int) env[env_key] = container_info return env[env_key]
def remove_chunks(self, env, chunks_names, container): error = False self.app.logger.info('StackSync API: internal remove uploaded chunks: container: %s', str(container)) for chunk_name in chunks_names: env_aux = env.copy() new_path = "/v1/" + env['stacksync_user_account'] + "/" + container + "/" + str(chunk_name) del env_aux['HTTP_STACKSYNC_API'] seg_req = make_pre_authed_request(env_aux, method='DELETE', path=new_path, agent=str(container)) seg_resp = seg_req.get_response(self.app) if not is_valid_status(seg_resp.status_int): self.app.logger.error('StackSync API: remove_chunks: error deleting uploaded chunks %s', str(chunk_name)) error = True break if error: self.app.logger.error( 'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend', seg_resp.status) return False return True
def test_pre_auth_req_with_quoted_path(self): r = wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'}, 'PUT', path=quote('/a space'), body='tester', headers={}) self.assertEquals(r.path, quote('/a space'))
def get_account_info(env, app, swift_source=None): """ Get the info structure for an account, based on env and app. This is useful to middlewares. Note: This call bypasses auth. Success does not imply that the request has authorization to the account_info. """ cache = cache_from_env(env) if not cache: return None (version, account, _junk, _junk) = \ split_path(env['PATH_INFO'], 2, 4, True) cache_key = get_account_memcache_key(account) # Use a unique environment cache key per account. If you copy this env # to make a new request, it won't accidentally reuse the old account info env_key = 'swift.%s' % cache_key if env_key not in env: account_info = cache.get(cache_key) if not account_info: resp = make_pre_authed_request( env, 'HEAD', '/%s/%s' % (version, account), swift_source=swift_source, ).get_response(app) account_info = headers_to_account_info(resp.headers, resp.status_int) env[env_key] = account_info return env[env_key]
def upload_file_chunks(self, env, chunked_file, container): error = False self.app.logger.info('StackSync API: upload_file_chunks: container: %s', str(container)) upload_chunks = [] for i in range(len(chunked_file.chunks)): chunk_name = chunked_file.name_list[i-1] chunk_content = chunked_file.chunks[i-1] env_aux = env.copy() new_path = "/v1/" + env['stacksync_user_account'] + "/" + container + "/" + chunk_name del env_aux['HTTP_STACKSYNC_API'] seg_req = make_pre_authed_request(env_aux, method='PUT', path=new_path, body=chunk_content, agent=str(container)) seg_resp = seg_req.get_response(self.app) if not is_valid_status(seg_resp.status_int): self.app.logger.error('StackSync API: upload_file_chunks: error uploading chunk %s', chunk_name) error = True break upload_chunks.append(chunk_name) if error: self.app.logger.error( 'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend', seg_resp.status) response = create_error_response(500, "Error uploading chunks to storage backend") self.remove_chunks(env, upload_chunks, container) else: response = HTTPCreated() return response
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env): marker = '' while True: lreq = make_pre_authed_request( env, method='GET', swift_source='VW', path='/v1/%s/%s' % (account_name, lcontainer)) lreq.environ['QUERY_STRING'] = \ 'format=json&prefix=%s&marker=%s' % (quote(lprefix), quote(marker)) lresp = lreq.get_response(self.app) if not is_success(lresp.status_int): if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif is_client_error(lresp.status_int): raise HTTPPreconditionFailed() else: raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break marker = sublisting[-1]['name'].encode('utf-8') yield sublisting
def get_account_info(env, app, swift_source=None): """ Get the info structure for an account, based on env and app. This is useful to middlewares. Note: This call bypasses auth. Success does not imply that the request has authorization to the account_info. """ cache = cache_from_env(env) if not cache: return None (version, account, _junk, _junk) = \ split_path(env['PATH_INFO'], 2, 4, True) cache_key = get_account_memcache_key(account) # Use a unique environment cache key per account. If you copy this env # to make a new request, it won't accidentally reuse the old account info env_key = 'swift.%s' % cache_key if env_key not in env: account_info = cache.get(cache_key) if not account_info: resp = make_pre_authed_request( env, 'HEAD', '/%s/%s' % (version, account), swift_source=swift_source, ).get_response(app) account_info = headers_to_account_info( resp.headers, resp.status_int) env[env_key] = account_info return env[env_key]
def handle_prep(self, req): """Prepare the backing store Swiftcluster for use with the auth system. Required headers: - `x-s3auth-prep-key`: must be same as key in config - `x-s3auth-hash-key`: hash key used for hashing admin key - `x-s3auth-admin-key`: admin key Note: The call can also be used to change current s3auth-admin key. """ prep_key = req.headers.get("x-s3auth-prep-key") hash_key = req.headers.get("x-s3auth-hash-key") admin_key = req.headers.get('x-s3auth-admin-key') if not all((prep_key, hash_key, admin_key)): return HTTPBadRequest( body='Headers x-s3auth-prep-key, x-s3auth-hash-key, ' 'x-s3auth-admin-key all required', request=req) if self.prep_key != prep_key: return _denied_response(req) hashed_admin_key = _hash_msg(admin_key, hash_key) path = quote('/v1/{}'.format(self.auth_account)) resp = make_pre_authed_request(req.environ, 'PUT', path, headers={ HKEY_HASH_KEY: hash_key, HKEY_HASHED_ADMIN_KEY: hashed_admin_key, }).get_response(self.app) if resp.status_int // 100 != 2: raise Exception('Could not PUT auth account: {} {}'.format( path, resp.status)) path = quote(self.akd_container_url) resp = make_pre_authed_request(req.environ, 'PUT', path).get_response(self.app) if resp.status_int // 100 != 2: raise Exception( 'Could not PUT access key details container: {} {}'.format( path, resp.status)) return HTTPOk(request=req)
def test_pre_auth_req_swift_source(self): r = wsgi.make_pre_authed_request({'QUERY_STRING': 'original'}, 'GET', 'path', 'the body', swift_source='UT') self.assertEquals(r.body, 'the body') self.assertEquals(r.environ['swift.source'], 'UT')
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env, marker='', end_marker='', reverse=True): '''Get "pages" worth of objects that start with a prefix. The optional keyword arguments ``marker``, ``end_marker``, and ``reverse`` are used similar to how they are for containers. We're either coming: - directly from ``_listing_iter``, in which case none of the optional args are specified, or - from ``_in_proxy_reverse_listing``, in which case ``reverse`` is ``False`` and both ``marker`` and ``end_marker`` are specified (although they may still be blank). ''' while True: lreq = make_pre_authed_request( env, method='GET', swift_source='VW', path='/v1/%s/%s' % (account_name, lcontainer)) lreq.environ['QUERY_STRING'] = \ 'format=json&prefix=%s&marker=%s' % ( quote(lprefix), quote(marker)) if end_marker: lreq.environ['QUERY_STRING'] += '&end_marker=%s' % ( quote(end_marker)) if reverse: lreq.environ['QUERY_STRING'] += '&reverse=on' lresp = lreq.get_response(self.app) if not is_success(lresp.status_int): close_if_possible(lresp.app_iter) if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif is_client_error(lresp.status_int): raise HTTPPreconditionFailed() else: raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break # When using the ``reverse`` param, check that the listing is # actually reversed first_item = sublisting[0]['name'].encode('utf-8') last_item = sublisting[-1]['name'].encode('utf-8') page_is_after_marker = marker and first_item > marker if reverse and (first_item < last_item or page_is_after_marker): # Apparently there's at least one pre-2.6.0 container server yield self._in_proxy_reverse_listing( account_name, lcontainer, lprefix, env, marker, sublisting) return marker = last_item yield sublisting
def origin_db_delete(self, env, req): """ Handles DELETEs in the Origin database. This is not really a delete- it will remove the object from the container listing and set cdn_enabled=false and a deleted flag in the .hash_* obj that says that the obj is deleted. This way the container won't show up in the listings, HEAD to the object will return 404s but behind the scenes lookups to the object will be able to determine the account and container from a container_hash. """ try: vsn, account, container = split_path(req.path, 3, 3) except ValueError: return HTTPBadRequest( 'Invalid request. ' 'URI format: /<api version>/<account>/<container>') if self.extra_header_for_deletes and not req.headers.get( self.extra_header_for_deletes, 'f').lower() in TRUE_VALUES: # only do delete if header is set (assuming you want the header) return HTTPMethodNotAllowed(request=req) hsh = self.hash_path(account, container) cdn_obj_path = self.get_hsh_obj_path(hsh) # Remove memcache entry memcache_client = utils.cache_from_env(env) if memcache_client: memcache_key = self.cdn_data_memcache_key(cdn_obj_path) memcache_client.delete(memcache_key) ref_hash_data = HashData(account, container, self.default_ttl, False, False, deleted=True) self._set_hash_data(env, cdn_obj_path, ref_hash_data, update_listings=False) cdn_list_path = quote('/v1/%s/%s/%s' % (self.origin_account, account, container)) list_resp = make_pre_authed_request(env, 'DELETE', cdn_list_path, agent='SwiftOrigin', swift_source='SOS').get_response( self.app) if list_resp.status_int // 100 != 2 and list_resp.status_int != 404: raise OriginDbFailure('Could not DELETE listing path in origin ' 'db: %s %s' % (cdn_list_path, list_resp.status_int)) # Return 404 if container didn't exist if list_resp.status_int == 404: return HTTPNotFound(request=req) return HTTPNoContent(request=req)
def _listing_pages_iter(self, account_name, lcontainer, lprefix, req, marker='', end_marker='', reverse=True): '''Get "pages" worth of objects that start with a prefix. The optional keyword arguments ``marker``, ``end_marker``, and ``reverse`` are used similar to how they are for containers. We're either coming: - directly from ``_listing_iter``, in which case none of the optional args are specified, or - from ``_in_proxy_reverse_listing``, in which case ``reverse`` is ``False`` and both ``marker`` and ``end_marker`` are specified (although they may still be blank). ''' while True: lreq = make_pre_authed_request( req.environ, method='GET', swift_source='VW', path=wsgi_quote('/v1/%s/%s' % (account_name, lcontainer))) lreq.environ['QUERY_STRING'] = \ 'prefix=%s&marker=%s' % (wsgi_quote(lprefix), wsgi_quote(marker)) if end_marker: lreq.environ['QUERY_STRING'] += '&end_marker=%s' % ( wsgi_quote(end_marker)) if reverse: lreq.environ['QUERY_STRING'] += '&reverse=on' lresp = lreq.get_response(self.app) if not is_success(lresp.status_int): close_if_possible(lresp.app_iter) if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif is_client_error(lresp.status_int): raise HTTPPreconditionFailed(request=req) else: raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break # When using the ``reverse`` param, check that the listing is # actually reversed first_item = bytes_to_wsgi(sublisting[0]['name'].encode('utf-8')) last_item = bytes_to_wsgi(sublisting[-1]['name'].encode('utf-8')) page_is_after_marker = marker and first_item > marker if reverse and (first_item < last_item or page_is_after_marker): # Apparently there's at least one pre-2.6.0 container server yield self._in_proxy_reverse_listing( account_name, lcontainer, lprefix, req, marker, sublisting) return marker = last_item yield sublisting
def swift_account(self, env, tenant_id): path = '/v1/%s/%s?format=json&prefix=account/' \ % (self.aggregate_account, tenant_id) req = make_pre_authed_request(env, 'GET', path) req.environ['swift.proxy_access_log_made'] = True resp = req.get_response(self.app) if resp.status_int == 404: return None return json.loads(resp.body)[0]['name'].split('/')[1]
def get_quota(self, env, version, account): """ Get quota und currently used storage from account """ request = make_pre_authed_request(env, 'HEAD', '/%s/%s' % (version, account)) response = request.get_response(self.app) bytes_used = response.headers.get('x-account-bytes-used', 0) quota = response.headers.get('x-account-meta-quota-bytes', -1) return (bytes_used, quota)
def origin_db_delete(self, env, req): """ Handles DELETEs in the Origin database """ if not self.delete_enabled: return HTTPMethodNotAllowed(request=req) try: vsn, account, container = split_path(req.path, 3, 3) except ValueError: return HTTPBadRequest( 'Invalid request. ' 'URI format: /<api version>/<account>/<container>') hsh = self.hash_path(account, container) cdn_obj_path = self.get_hsh_obj_path(hsh) # Remove memcache entry memcache_client = utils.cache_from_env(env) if memcache_client: memcache_key = self.cdn_data_memcache_key(cdn_obj_path) memcache_client.delete(memcache_key) resp = make_pre_authed_request( env, 'DELETE', cdn_obj_path, agent='SwiftOrigin', swift_source='SOS').get_response(self.app) # A 404 means it's already deleted, which is okay if resp.status_int // 100 != 2 and resp.status_int != 404: raise OriginDbFailure( 'Could not DELETE .hash obj in origin ' 'db: %s %s' % (cdn_obj_path, resp.status_int)) cdn_list_path = quote('/v1/%s/%s/%s' % (self.origin_account, account, container)) list_resp = make_pre_authed_request( env, 'DELETE', cdn_list_path, agent='SwiftOrigin', swift_source='SOS').get_response(self.app) if list_resp.status_int // 100 != 2 and list_resp.status_int != 404: raise OriginDbFailure( 'Could not DELETE listing path in origin ' 'db: %s %s' % (cdn_list_path, list_resp.status_int)) # Return 404 if container didn't exist if resp.status_int == 404 and list_resp.status_int == 404: return HTTPNotFound(request=req) return HTTPNoContent(request=req)
def _put_versioned_obj(self, req, put_path_info, source_resp): # Create a new Request object to PUT to the versions container, copying # all headers from the source object apart from x-timestamp. put_req = make_pre_authed_request( req.environ, path=put_path_info, method='PUT', swift_source='VW') copy_header_subset(source_resp, put_req, lambda k: k.lower() != 'x-timestamp') put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) return put_req.get_response(self.app)
def get_account_info(self, env, account): path = '/v1/%s' % account req = make_pre_authed_request(env, 'HEAD', path) req.environ['swift.proxy_access_log_made'] = True resp = req.get_response(self.app) if not resp.status_int // 100 == 2: return (0, 0, 0) return (int(resp.headers.get('x-account-container-count', 0)), int(resp.headers.get('x-account-object-count', 0)), int(resp.headers.get('x-account-bytes-used', 0)))
def __call__(self, request): try: (version, account, container, objname) = split_path( request.path_info, 1, 4, True) except ValueError: return self.app(environ, start_response) if container and not objname: if request.method in ('DELETE', 'HEAD'): return self.app if request.method == 'POST': # Deny setting if there are any objects in base container # Otherwise these objects won't be visible if request.headers.get('X-Container-Meta-Storage-Path'): container_info = get_container_info(request.environ, self.app) objects = container_info.get('object_count') if objects and int(objects) > 0: return HTTPBadRequest() # ACL set groups = (request.remote_user or '').split(',') account_name = groups[0].split(':')[0] read_acl = request.environ.get('HTTP_X_CONTAINER_READ', '') for target_account in read_acl.split(','): target_account = target_account.split(':')[0] target_storage_path = self._get_storage_path(request, target_account) if not target_storage_path or account_name == target_account: continue container_path = "/%s/%s/%s" % (version, account, container) headers = {'X-Container-Meta-Storage-Path': container_path} request_path = "%s/%s%s_%s" % (target_storage_path, self.prefix, account_name, container) req = make_pre_authed_request(request.environ, 'PUT', request_path, headers=headers) req.get_response(self.app) if container: container_info = get_container_info(request.environ, self.app) meta = container_info.get('meta', {}) storage_path = meta.get('storage-path') if storage_path: if objname: storage_path += '/' + objname request.environ['PATH_INFO'] = storage_path request.environ['RAW_PATH_INFO'] = storage_path return self.app
def _post_slomd5_header(self, env, SLOmd5): headers = {} headers['X-Object-Meta-SLOmd5'] = SLOmd5 post_req = make_pre_authed_request(env, method='POST', swift_source='SLOmd5', path=env['PATH_INFO'], headers=headers) post_resp = post_req.get_response(self.app) if not is_success(post_resp.status_int): self.logger.info('POST with SLOmd5 header failed: ' + str(post_resp.body))
def handle_request(self, env, req): """ Handles the POST /origin/.prep call for preparing the backing store Swift cluster for use with the origin subsystem. Can only be called by .origin_admin :param req: The swob.Request to process. :returns: swob.Response, 204 on success """ if not self.is_origin_admin(req): return HTTPForbidden(request=req) try: vsn, account = split_path(req.path, 2, 2) except ValueError: return HTTPBadRequest(request=req) if account == '.prep': path = '/v1/%s' % self.origin_account resp = make_pre_authed_request(req.environ, 'PUT', path, agent='SwiftOrigin', swift_source='SOS').get_response( self.app) if resp.status_int // 100 != 2: raise Exception( 'Could not create the main origin account: %s %s' % (path, resp.status)) for i in xrange(self.num_hash_cont): cont_name = '.hash_%d' % i path = '/v1/%s/%s' % (self.origin_account, cont_name) resp = make_pre_authed_request( req.environ, 'PUT', path, agent='SwiftOrigin', swift_source='SOS').get_response(self.app) if resp.status_int // 100 != 2: raise Exception('Could not create %s container: %s %s' % (cont_name, path, resp.status)) return HTTPNoContent(request=req) return HTTPNotFound(request=req)
def _put_versioned_obj(self, req, put_path_info, source_resp): # Create a new Request object to PUT to the versions container, copying # all headers from the source object apart from x-timestamp. put_req = make_pre_authed_request( req.environ, path=quote(put_path_info), method='PUT', swift_source='VW') copy_header_subset(source_resp, put_req, lambda k: k.lower() != 'x-timestamp') put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) put_resp = put_req.get_response(self.app) close_if_possible(source_resp.app_iter) return put_resp
def list_containers_iter(self, *args, **kwargs): """ Returns a list of containers the user has access to """ try: version, account = self.request.split_path(2, 2) except ValueError: pass path = "/%s/%s?format=json" % (version, account) for key in ('limit', 'marker', 'end_marker', 'prefix', 'delimiter'): value = self.request.params.get(key) if value: path+= '&%s=%s' % (key, value) if self.memcache_client is None: self.memcache_client = cache_from_env(self.request.environ) memcache_key = 'containerlist%s%s' % (path, str(self.groups)) containers = self.memcache_client.get(memcache_key) if containers is not None: return containers req = make_pre_authed_request(self.request.environ, 'GET', path) resp = req.get_response(self.app) tmp_containers = json.loads(resp.body) # No cached result? -> ratelimit request to prevent abuse memcache_key_sleep = 'containerlist_sleep/%s' % self.account last_request_time = self.memcache_client.get(memcache_key_sleep) if last_request_time and len(tmp_containers) > 0: last_request = time.time() - last_request_time if last_request < self.min_sleep: eventlet.sleep(self.min_sleep - last_request) containers = [] for container in tmp_containers: tmp_env = copy.copy(self.request.environ) container_name = container['name'].encode("utf8") path_info = "/%s/%s/%s" % (version, account, container_name) tmp_env['PATH_INFO'] = path_info container_info = get_container_info(tmp_env, self.app) acl = (container_info.get('read_acl') or '').split(',') if (list(set(self.groups) & set(acl))): containers.append((container['name'], container['count'], container['bytes'], 0)) self.memcache_client.set(memcache_key, containers, time=60) self.memcache_client.set(memcache_key_sleep, time.time()) return containers
def _get_quota(self, account, env): """ Get quota und currently used storage from account """ request = make_pre_authed_request(env, 'HEAD', '/v1/' + account) response = request.get_response(self.app) quota = -1 for (key, value) in response.headers.items(): if key == 'x-account-bytes-used': bytes_used = int(value) if key == 'x-account-meta-bytes-limit': quota = int(value) return (bytes_used, quota)
def _get_container_info(self, env): """ Retrieves x-container-meta-web-index, x-container-meta-web-error, x-container-meta-web-listings, x-container-meta-web-listings-css, and x-container-meta-web-directory-type from memcache or from the cluster and stores the result in memcache and in self._index, self._error, self._listings, self._listings_css and self._dir_type. :param env: The WSGI environment dict. """ self._index = self._error = self._listings = self._listings_css = \ self._dir_type = None memcache_client = cache_from_env(env) if memcache_client: cached_data = memcache_client.get( get_memcache_key(self.version, self.account, self.container)) if cached_data: (self._index, self._error, self._listings, self._listings_css, self._dir_type) = cached_data return else: cached_data = memcache_client.get( get_compat_memcache_key( self.version, self.account, self.container)) if cached_data: (self._index, self._error, self._listings, self._listings_css) = cached_data self._dir_type = '' return resp = make_pre_authed_request( env, 'HEAD', '/%s/%s/%s' % ( self.version, self.account, self.container), agent=self.agent, swift_source='SW').get_response(self.app) if is_success(resp.status_int): self._index = \ resp.headers.get('x-container-meta-web-index', '').strip() self._error = \ resp.headers.get('x-container-meta-web-error', '').strip() self._listings = \ resp.headers.get('x-container-meta-web-listings', '').strip() self._listings_css = \ resp.headers.get('x-container-meta-web-listings-css', '').strip() self._dir_type = \ resp.headers.get('x-container-meta-web-directory-type', '').strip() if memcache_client: memcache_client.set( get_memcache_key( self.version, self.account, self.container), (self._index, self._error, self._listings, self._listings_css, self._dir_type), time=self.cache_timeout)
def handle_delete(self, req, start_response): """ Handle request to delete a user's container. As part of deleting a container, this middleware will also delete the hidden container holding object versions. Before a user's container can be deleted, swift must check if there are still old object versions from that container. Only after disabling versioning and deleting *all* object versions can a container be deleted. """ container_info = get_container_info(req.environ, self.app, swift_source='OV') versions_cont = unquote( container_info.get('sysmeta', {}).get('versions-container', '')) if versions_cont: account = req.split_path(3, 3, True)[1] # using a HEAD request here as opposed to get_container_info # to make sure we get an up-to-date value versions_req = make_pre_authed_request( req.environ, method='HEAD', swift_source='OV', path=wsgi_quote('/v1/%s/%s' % (account, str_to_wsgi(versions_cont))), headers={'X-Backend-Allow-Reserved-Names': 'true'}) vresp = versions_req.get_response(self.app) drain_and_close(vresp) if vresp.is_success and int( vresp.headers.get('X-Container-Object-Count', 0)) > 0: raise HTTPConflict( 'Delete all versions before deleting container.', request=req) elif not vresp.is_success and vresp.status_int != 404: raise HTTPInternalServerError( 'Error deleting versioned container') else: versions_req.method = 'DELETE' resp = versions_req.get_response(self.app) drain_and_close(resp) if not is_success(resp.status_int) and resp.status_int != 404: raise HTTPInternalServerError( 'Error deleting versioned container') app_resp = self._app_call(req.environ) start_response(self._response_status, self._response_headers, self._response_exc_info) return app_resp
def get_cdn_data(self, env, cdn_obj_path): """ Retrieves HashData object from memcache or by doing a GET of the cdn_obj_path which should be what is returned from get_hsh_obj_path. Will return None if the HashData is "deleted" :returns: HashData object or None. """ memcache_client = utils.cache_from_env(env) memcache_key = self.cdn_data_memcache_key(cdn_obj_path) if memcache_client: cached_cdn_data = memcache_client.get(memcache_key) if cached_cdn_data == '404': return None if cached_cdn_data: try: hash_data = HashData.create_from_json(cached_cdn_data) if hash_data.deleted: return None else: return hash_data except ValueError: pass resp = make_pre_authed_request(env, 'GET', cdn_obj_path, agent='SwiftOrigin', swift_source='SOS').get_response( self.app) if resp.status_int // 100 == 2: try: if memcache_client: memcache_client.set(memcache_key, resp.body, serialize=False, time=MEMCACHE_TIMEOUT) hash_data = HashData.create_from_json(resp.body) if not hash_data.deleted: return hash_data except ValueError: self.logger.warn('Invalid HashData json: %s' % cdn_obj_path) if resp.status_int == 404: if memcache_client: # only memcache for 30 secs in case adding container to swift memcache_client.set(memcache_key, '404', serialize=False, time=CACHE_404) return None
def update_last_modified_meta(self, req, env): vrs, account, container, obj = req.split_path(1, 4, True) path = env['PATH_INFO'] if obj: path = path.split('/%s' % obj)[0] metakey = 'X-Container-Meta-%s' % self.key_name headers = {metakey: str(time.time())} set_meta_req = wsgi.make_pre_authed_request(env, method='POST', path=path, headers=headers, swift_source='lm') set_meta_req.get_response(self.app)
def get_data_from_url(url, app, key, logger, env): if not url or not key: return None req = make_pre_authed_request(env, method='GET', path='%s/%s' % (url, quote(key)), swift_source='liteauth', agent='Liteauth') resp = req.get_response(app) if resp.status_int >= 300: logger.info('get_data response for %s is %s %s' % (req.path, resp.status, resp.body)) return None return resp.body.strip()
def _get_source_object(self, req, path_info): # make a pre_auth request in case the user has write access # to container, but not READ. This was allowed in previous version # (i.e., before middleware) so keeping the same behavior here get_req = make_pre_authed_request( req.environ, path=path_info, headers={'X-Newest': 'True'}, method='GET', swift_source='VW') source_resp = get_req.get_response(self.app) if source_resp.content_length is None or \ source_resp.content_length > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) return source_resp
def handle_get_listing(self, req): """Retrieve a new-line separated list of all access keys. Required headers: - `x-s3auth-admin-key`: admin key """ path = quote(self.akd_container_url) resp = make_pre_authed_request(req.environ, 'GET', path).get_response(self.app) if resp.status_int // 100 == 2: listing = '\n'.join([e['name'] for e in json.loads(resp.body)]) return HTTPOk(request=req, body=listing) else: raise Exception('Could not GET access key listing: {} {}'.format( path, resp.status_int))
def store_data_in_url(url, app, key, data, env): if not url or not key: return False req = make_pre_authed_request(env, method='PUT', path='%s/%s' % (url, quote(key)), body=str(data), headers={'content-type': 'text/plain'}, swift_source='liteauth', agent='Liteauth') req.environ['liteauth.new_service'] = \ env.get('liteauth.new_service', None) resp = req.get_response(app) if resp.status_int >= 300: return False return True
def _put_versioned_obj(self, req, put_path_info, source_resp): # Create a new Request object to PUT to the container, copying # all headers from the source object apart from x-timestamp. put_req = make_pre_authed_request( req.environ, path=wsgi_quote(put_path_info), method='PUT', swift_source='VW') copy_header_subset(source_resp, put_req, lambda k: k.lower() != 'x-timestamp') slo_size = put_req.headers.get('X-Object-Sysmeta-Slo-Size') if slo_size: put_req.headers['Content-Type'] += '; swift_bytes=' + slo_size put_req.environ['swift.content_type_overridden'] = True put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) put_resp = put_req.get_response(self.app) close_if_possible(source_resp.app_iter) return put_resp
def __call__(self, env, start_response): if env['REQUEST_METHOD'] != 'PUT': return self.app(env, start_response) version, account, container, obj = split_path(env['PATH_INFO'], 1, 4, True) if not obj: return self.app(env, start_response) if not obj.lower().endswith('.jpg') and\ not obj.lower().endswith('.jpeg'): return self.app(env, start_response) extractor = MetaExtractor(env['wsgi.input']) env['wsgi.input'] = extractor try: resp = self._app_call(env) except Exception: resp = HTTPServerError(request=Request(env), body="error") return resp(env, start_response) start_response(self._response_status, self._response_headers, self._response_exc_info) status = int(self._response_status.split()[0]) if status < 200 or status > 300: return resp headers = {} for tag in extractor.exif_parser.tags: if tag.tag().startswith('Unknown'): continue if len(tag.value()) > MAX_META_VALUE_LENGTH: continue header = 'X-Object-Meta-' + tag.tag().replace(' ', '-') headers[header] = tag.value() # TODO: merge with the PUT headers self.logger.info(headers) post_req = make_pre_authed_request(env, method='POST', swift_source='JpegMeta', path=env['PATH_INFO'], headers=headers) post_resp = post_req.get_response(self.app) if not is_success(post_resp.status_int): self.logger.info('POST with JPEG headers failed: ' + str(post_resp.body)) return resp
def handle_obj_versions_delete_push(self, req, versions_cont, api_version, account_name, container_name, object_name): """ Handle DELETE requests when in history mode. Copy current version of object to versions_container and write a delete marker before proceeding with original request. :param req: original request. :param versions_cont: container where previous versions of the object are stored. :param api_version: api version. :param account_name: account name. :param object_name: name of object of original request """ self._copy_current(req, versions_cont, api_version, account_name, object_name) marker_path = "/%s/%s/%s/%s" % ( api_version, account_name, versions_cont, self._build_versions_object_name(object_name, time.time())) marker_headers = { # Definitive source of truth is Content-Type, and since we add # a swift_* param, we know users haven't set it themselves. # This is still open to users POSTing to update the content-type # but they're just shooting themselves in the foot then. 'content-type': DELETE_MARKER_CONTENT_TYPE, 'content-length': '0', 'x-auth-token': req.headers.get('x-auth-token') } marker_req = make_pre_authed_request(req.environ, path=wsgi_quote(marker_path), headers=marker_headers, method='PUT', swift_source='VW') marker_req.environ['swift.content_type_overridden'] = True marker_resp = marker_req.get_response(self.app) self._check_response_error(req, marker_resp) drain_and_close(marker_resp) # successfully copied and created delete marker; safe to delete return self.app
def _set_details(self, req, access_key, secret_key, account): """Set access key details.""" path = quote(self.akd_container_url + access_key) resp = make_pre_authed_request( env=req.environ, method='PUT', path=path, body=json.dumps({'secret_key': secret_key, 'account': account})).\ get_response(self.app) if resp.status_int // 100 == 2: # Remove old data from cache. memcache_client = cache_from_env(req.environ) if memcache_client: memcache_key = MEMCACHE_KEY_FORMAT % (self.reseller_prefix, access_key) memcache_client.delete(memcache_key) else: raise Exception('Could not PUT access key details: {} {}'.format( path, resp.status_int))
def _get_source_object(self, req, path_info): # make a pre_auth request in case the user has write access # to container, but not READ. This was allowed in previous version # (i.e., before middleware) so keeping the same behavior here get_req = make_pre_authed_request(req.environ, path=wsgi_quote(path_info) + '?symlink=get', headers={'X-Newest': 'True'}, method='GET', swift_source='VW') source_resp = get_req.get_response(self.app) if source_resp.content_length is None or \ source_resp.content_length > MAX_FILE_SIZE: # Consciously *don't* drain the response before closing; # any logged 499 is actually rather appropriate here close_if_possible(source_resp.app_iter) return HTTPRequestEntityTooLarge(request=req) return source_resp
def upload_file_chunks(self, env, chunked_file, container): error = False self.app.logger.info( 'StackSync API: upload_file_chunks: container: %s', str(container)) upload_chunks = [] for i in range(len(chunked_file.chunks)): chunk_name = chunked_file.name_list[i - 1] chunk_content = chunked_file.chunks[i - 1] env_aux = env.copy() new_path = "/v1/" + env[ 'stacksync_user_account'] + "/" + container + "/" + chunk_name del env_aux['HTTP_STACKSYNC_API'] seg_req = make_pre_authed_request(env_aux, method='PUT', path=new_path, body=chunk_content, agent=str(container)) seg_resp = seg_req.get_response(self.app) if not is_valid_status(seg_resp.status_int): self.app.logger.error( 'StackSync API: upload_file_chunks: error uploading chunk %s', chunk_name) error = True break upload_chunks.append(chunk_name) if error: self.app.logger.error( 'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend', seg_resp.status) response = create_error_response( 500, "Error uploading chunks to storage backend") self.remove_chunks(env, upload_chunks, container) else: response = HTTPCreated() return response