def test_bucket_GET_max_keys(self): bucket_name = 'junk' req = Request.blank('/%s?max-keys=5' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./MaxKeys').text, '5') _, path = self.swift.calls[-1] _, query_string = path.split('?') args = dict(cgi.parse_qsl(query_string)) self.assertEqual(args['limit'], '6') req = Request.blank('/%s?max-keys=5000' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./MaxKeys').text, '5000') _, path = self.swift.calls[-1] _, query_string = path.split('?') args = dict(cgi.parse_qsl(query_string)) self.assertEqual(args['limit'], '1001')
def test_bucket_acl_PUT(self): elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = 'id' acl = SubElement(elem, 'AccessControlList') grant = SubElement(acl, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = \ 'http://acs.amazonaws.com/groups/global/AllUsers' SubElement(grant, 'Permission').text = 'READ' xml = tostring(elem) req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac'}, body=xml) status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200') req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': StringIO(xml)}, headers={'Authorization': 'AWS test:tester:hmac', 'Transfer-Encoding': 'chunked'}) self.assertIsNone(req.content_length) self.assertIsNone(req.message_length()) status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200')
def test_proxy_client_logging(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={ 'REQUEST_METHOD': 'GET', 'REMOTE_ADDR': '1.2.3.4', 'HTTP_X_FORWARDED_FOR': '4.5.6.7,8.9.10.11'}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[0], '4.5.6.7') # client ip self.assertEquals(log_parts[1], '1.2.3.4') # remote addr app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={ 'REQUEST_METHOD': 'GET', 'REMOTE_ADDR': '1.2.3.4', 'HTTP_X_CLUSTER_CLIENT_IP': '4.5.6.7'}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[0], '4.5.6.7') # client ip self.assertEquals(log_parts[1], '1.2.3.4') # remote addr
def test_bucket_GET_v2_with_delimiter_max_keys(self): bucket_name = 'junk' req = Request.blank( '/%s?list-type=2&delimiter=a&max-keys=2' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') next_token = elem.find('./NextContinuationToken') self.assertIsNotNone(next_token) self.assertEqual(elem.find('./MaxKeys').text, '2') self.assertEqual(elem.find('./IsTruncated').text, 'true') req = Request.blank( '/%s?list-type=2&delimiter=a&max-keys=2&continuation-token=%s' % (bucket_name, next_token.text), environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') names = [o.find('./Key').text for o in elem.iterchildren('Contents')] self.assertEqual(names[0], 'lily')
def test_extract_tar_works(self): # On systems where $TMPDIR is long (like OS X), we need to do this # or else every upload will fail due to the path being too long. self.app.max_pathlen += len(self.testdir) for compress_format in ['', 'gz', 'bz2']: base_name = 'base_works_%s' % compress_format dir_tree = [ {base_name: [{'sub_dir1': ['sub1_file1', 'sub1_file2']}, {'sub_dir2': ['sub2_file1', u'test obj \u2661']}, 'sub_file1', {'sub_dir3': [{'sub4_dir1': '../sub4 file1'}]}, {'sub_dir4': None}, ]}] build_dir_tree(self.testdir, dir_tree) mode = 'w' extension = '' if compress_format: mode += ':' + compress_format extension += '.' + compress_format tar = tarfile.open(name=os.path.join(self.testdir, 'tar_works.tar' + extension), mode=mode) tar.add(os.path.join(self.testdir, base_name)) tar.close() req = Request.blank('/tar_works/acc/cont/') req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar' + extension)) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, compress_format) resp_data = utils.json.loads(resp_body) self.assertEquals(resp_data['Number Files Created'], 6) # test out xml req = Request.blank('/tar_works/acc/cont/') req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar' + extension)) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter( req, compress_format, 'application/xml') self.assert_('<response_status>201 Created</response_status>' in resp_body) self.assert_('<number_files_created>6</number_files_created>' in resp_body) # test out nonexistent format req = Request.blank('/tar_works/acc/cont/?extract-archive=tar', headers={'Accept': 'good_xml'}) req.environ['REQUEST_METHOD'] = 'PUT' req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar' + extension)) req.headers['transfer-encoding'] = 'chunked' def fake_start_response(*args, **kwargs): pass app_iter = self.bulk(req.environ, fake_start_response) resp_body = ''.join([i for i in app_iter]) self.assert_('Response Status: 406' in resp_body)
def test_get_account_info_cache(self): # Works with fake apps that return ints in the headers cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) resp = get_account_info(req.environ, FakeApp()) self.assertEqual(resp['bytes'], 3333) self.assertEqual(resp['total_object_count'], 10) self.assertEqual(resp['status'], 404) # Works with strings too, like you get when parsing HTTP headers # that came in through a socket from the account server cached = {'status': 404, 'bytes': '3333', 'container_count': '234', 'total_object_count': '10', 'meta': {}} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) resp = get_account_info(req.environ, FakeApp()) self.assertEqual(resp['status'], 404) self.assertEqual(resp['bytes'], 3333) self.assertEqual(resp['container_count'], 234) self.assertEqual(resp['meta'], {}) self.assertEqual(resp['total_object_count'], 10)
def test_policy_index(self): # Policy index can be specified by X-Backend-Storage-Policy-Index # in the request header for object API app = proxy_logging.ProxyLoggingMiddleware(FakeApp(policy_idx='1'), {}) app.access_logger = FakeLogger() req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) resp = app(req.environ, start_response) ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[20], '1') # Policy index can be specified by X-Backend-Storage-Policy-Index # in the response header for container API app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) def fake_call(app, env, start_response): start_response(app.response_str, [('Content-Type', 'text/plain'), ('Content-Length', str(sum(map(len, app.body)))), ('X-Backend-Storage-Policy-Index', '1')]) while env['wsgi.input'].read(5): pass return app.body with mock.patch.object(FakeApp, '__call__', fake_call): resp = app(req.environ, start_response) ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[20], '1')
def __call__(self, env, start_response): req = Request(env) try: (version, account, container, obj) = req.split_path(4, 4, True) except ValueError: # If obj component is not present in req, do not proceed further. return self.app(env, start_response) self.account_name = account self.container_name = container self.object_name = obj # Save off original request method (COPY/POST) in case it gets mutated # into PUT during handling. This way logging can display the method # the client actually sent. req.environ['swift.orig_req_method'] = req.method if req.method == 'PUT' and req.headers.get('X-Copy-From'): return self.handle_PUT(req, start_response) elif req.method == 'COPY': return self.handle_COPY(req, start_response) elif req.method == 'POST' and self.object_post_as_copy: return self.handle_object_post_as_copy(req, start_response) elif req.method == 'OPTIONS': # Does not interfere with OPTIONS response from (account,container) # servers and /info response. return self.handle_OPTIONS(req, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): # If override is set in env, then just pass along if config_true_value(env.get('swift.crypto.override')): return self.app(env, start_response) req = Request(env) if self.disable_encryption and req.method in ('PUT', 'POST'): return self.app(env, start_response) try: req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) if req.method in ('GET', 'HEAD'): handler = EncrypterObjContext(self, self.logger).handle_get_or_head elif req.method == 'PUT': handler = EncrypterObjContext(self, self.logger).handle_put elif req.method == 'POST': handler = EncrypterObjContext(self, self.logger).handle_post else: # anything else return self.app(env, start_response) try: return handler(req, start_response) except HTTPException as err_resp: return err_resp(env, start_response)
def test_check_signature_multi_bytes_secret_failure(self): # Test v2 check_signature with multi bytes invalid secret req = Request.blank('/photos/puppy.jpg', headers={ 'Host': 'johnsmith.s3.amazonaws.com', 'Date': 'Tue, 27 Mar 2007 19:36:42 +0000', 'Authorization': ('AWS AKIAIOSFODNN7EXAMPLE:' 'bWq2s1WEIj+Ydj0vQ697zp+IXMU='), }) sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com') # This is a failure case with utf-8 non-ascii multi-bytes charactor # but we expect to return just False instead of exceptions self.assertFalse(sigv2_req.check_signature( u'\u30c9\u30e9\u30b4\u30f3')) # Test v4 check_signature with multi bytes invalid secret amz_date_header = self.get_v4_amz_date_header() req = Request.blank('/photos/puppy.jpg', headers={ 'Authorization': 'AWS4-HMAC-SHA256 ' 'Credential=test/%s/us-east-1/s3/aws4_request, ' 'SignedHeaders=host;x-amz-content-sha256;x-amz-date,' 'Signature=X' % amz_date_header.split('T', 1)[0], 'X-Amz-Content-SHA256': '0123456789', 'X-Amz-Date': amz_date_header }) sigv4_req = SigV4Request( req.environ, storage_domain='s3.amazonaws.com') self.assertFalse(sigv4_req.check_signature( u'\u30c9\u30e9\u30b4\u30f3'))
def test_proxy_client_logging(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank( "/", environ={"REQUEST_METHOD": "GET", "REMOTE_ADDR": "1.2.3.4", "HTTP_X_FORWARDED_FOR": "4.5.6.7,8.9.10.11"}, ) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[0], "4.5.6.7") # client ip self.assertEquals(log_parts[1], "1.2.3.4") # remote addr app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank( "/", environ={"REQUEST_METHOD": "GET", "REMOTE_ADDR": "1.2.3.4", "HTTP_X_CLUSTER_CLIENT_IP": "4.5.6.7"} ) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[0], "4.5.6.7") # client ip self.assertEquals(log_parts[1], "1.2.3.4") # remote addr
def test_PUT_encryption_override(self): # set crypto override to disable encryption. # simulate another middleware wanting to set footers other_footers = { 'Etag': 'other etag', 'X-Object-Sysmeta-Other': 'other sysmeta', 'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override'} body = 'FAKE APP' env = {'REQUEST_METHOD': 'PUT', 'swift.crypto.override': True, 'swift.callback.update_footers': lambda footers: footers.update(other_footers)} hdrs = {'content-type': 'text/plain', 'content-length': str(len(body))} req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs) self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) resp = req.get_response(self.encrypter) self.assertEqual('201 Created', resp.status) # verify that other middleware's footers made it to app req_hdrs = self.app.headers[0] for k, v in other_footers.items(): self.assertEqual(v, req_hdrs[k]) # verify object is NOT encrypted by getting direct from the app get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.assertEqual(body, get_req.get_response(self.app).body)
def test_to_swift_req_subrequest_proxy_access_log(self): container = 'bucket' obj = 'obj' method = 'GET' # force_swift_request_proxy_log is True req = Request.blank('/%s/%s' % (container, obj), environ={'REQUEST_METHOD': method, 'swift.proxy_access_log_made': True}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) with patch.object(Request, 'get_response') as m_swift_resp, \ patch.object(Request, 'remote_user', 'authorized'): m_swift_resp.return_value = FakeSwiftResponse() s3_req = S3AclRequest( req.environ, MagicMock(), force_request_log=True) sw_req = s3_req.to_swift_req(method, container, obj) self.assertFalse(sw_req.environ['swift.proxy_access_log_made']) # force_swift_request_proxy_log is False req = Request.blank('/%s/%s' % (container, obj), environ={'REQUEST_METHOD': method, 'swift.proxy_access_log_made': True}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) with patch.object(Request, 'get_response') as m_swift_resp, \ patch.object(Request, 'remote_user', 'authorized'): m_swift_resp.return_value = FakeSwiftResponse() s3_req = S3AclRequest( req.environ, MagicMock(), force_request_log=False) sw_req = s3_req.to_swift_req(method, container, obj) self.assertTrue(sw_req.environ['swift.proxy_access_log_made'])
def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) # install our COPY-callback hook env['swift.copy_hook'] = self.copy_hook( env.get('swift.copy_hook', lambda src_req, src_resp, sink_req: src_resp)) try: if req.method == 'PUT' and \ req.params.get('multipart-manifest') == 'put': return self.handle_multipart_put(req, start_response) if req.method == 'DELETE' and \ req.params.get('multipart-manifest') == 'delete': return self.handle_multipart_delete(req)(env, start_response) if req.method == 'GET' or req.method == 'HEAD': return self.handle_multipart_get_or_head(req, start_response) if 'X-Static-Large-Object' in req.headers: raise HTTPBadRequest( request=req, body='X-Static-Large-Object is a reserved header. ' 'To create a static large object add query param ' 'multipart-manifest=put.') except HTTPException as err_resp: return err_resp(env, start_response) return self.app(env, start_response)
def test_check_object_creation_copy(self): headers = {'Content-Length': '0', 'X-Copy-From': 'c/o2', 'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation(Request.blank( '/', headers=headers), 'object_name'), None) headers = {'Content-Length': '1', 'X-Copy-From': 'c/o2', 'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation(Request.blank( '/', headers=headers), 'object_name').status_int, HTTP_BAD_REQUEST) headers = {'Transfer-Encoding': 'chunked', 'X-Copy-From': 'c/o2', 'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation(Request.blank( '/', headers=headers), 'object_name'), None) # a content-length header is always required headers = {'X-Copy-From': 'c/o2', 'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation(Request.blank( '/', headers=headers), 'object_name').status_int, HTTP_LENGTH_REQUIRED)
def test_GETorHEAD_base(self): base = Controller(self.app) req = Request.blank('/v1/a/c/o/with/slashes') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', '/a/c/o/with/slashes') self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ) self.assertEqual( resp.environ['swift.object/a/c/o/with/slashes']['status'], 200) req = Request.blank('/v1/a/c/o') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', '/a/c/o') self.assertTrue('swift.object/a/c/o' in resp.environ) self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200) req = Request.blank('/v1/a/c') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): resp = base.GETorHEAD_base(req, 'container', FakeRing(), 'part', '/a/c') self.assertTrue('swift.container/a/c' in resp.environ) self.assertEqual(resp.environ['swift.container/a/c']['status'], 200) req = Request.blank('/v1/a') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): resp = base.GETorHEAD_base(req, 'account', FakeRing(), 'part', '/a') self.assertTrue('swift.account/a' in resp.environ) self.assertEqual(resp.environ['swift.account/a']['status'], 200)
def test_handle_multipart_put_bad_data(self): bad_data = json.dumps([{"path": "/cont/object", "etag": "etagoftheobj", "size_bytes": "lala"}]) req = Request.blank( "/test_good/AUTH_test/c/man?multipart-manifest=put", environ={"REQUEST_METHOD": "PUT"}, body=bad_data ) self.assertRaises(HTTPException, self.slo.handle_multipart_put, req) for bad_data in [ json.dumps([{"path": "/cont", "etag": "etagoftheobj", "size_bytes": 100}]), json.dumps("asdf"), json.dumps(None), json.dumps(5), "not json", "1234", None, "", json.dumps({"path": None}), json.dumps([{"path": "/c/o", "etag": None, "size_bytes": 12}]), json.dumps([{"path": "/c/o", "etag": "asdf", "size_bytes": "sd"}]), json.dumps([{"path": 12, "etag": "etagoftheobj", "size_bytes": 100}]), json.dumps([{"path": u"/cont/object\u2661", "etag": "etagoftheobj", "size_bytes": 100}]), json.dumps([{"path": 12, "size_bytes": 100}]), json.dumps([{"path": 12, "size_bytes": 100}]), json.dumps([{"path": None, "etag": "etagoftheobj", "size_bytes": 100}]), ]: req = Request.blank( "/test_good/AUTH_test/c/man?multipart-manifest=put", environ={"REQUEST_METHOD": "PUT"}, body=bad_data ) self.assertRaises(HTTPException, self.slo.handle_multipart_put, req)
def __call__(self, env, start_response): if config_true_value(env.get('swift.crypto.override')): return self.app(env, start_response) req = Request(env) if self.disable_encryption and req.method in ('PUT', 'POST'): return self.app(env, start_response) try: req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) fetch_crypto_keys = env.get(CRYPTO_KEY_CALLBACK) if fetch_crypto_keys is not None: try: fetch_crypto_keys() except HTTPException as exc: if MISSING_KEY_MSG in exc.body: if req.method in ('PUT', 'POST'): # No key, just upload without encryption env['swift.crypto.override'] = True # else: # let the thing fail later, # if a key is required for decoding else: raise except Exception: # Let the parent class handle other exceptions pass res = super(Encrypter, self).__call__(env, start_response) return res
def test_get_account_info_cache(self): # The original test that we prefer to preserve cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['total_object_count'], 10) self.assertEquals(resp['status'], 404) # Here is a more realistic test cached = {'status': 404, 'bytes': '3333', 'container_count': '234', 'total_object_count': '10', 'meta': {}} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['status'], 404) self.assertEquals(resp['bytes'], '3333') self.assertEquals(resp['container_count'], 234) self.assertEquals(resp['meta'], {}) self.assertEquals(resp['total_object_count'], '10')
def test_parse_account_that_looks_like_version(self): """ Demonstrate the failure mode for accounts that looks like versions, if you can make _parse_path better and this is the *only* test that fails you can delete it ;) """ bad_accounts = ( 'v3.0', 'verybaddaccountwithnoprefix', ) for bad_account in bad_accounts: req = Request.blank('/endpoints/%s/c/o' % bad_account) self.assertRaises(ValueError, self.list_endpoints._parse_path, req) even_worse_accounts = { 'v1': 1.0, 'v2.0': 2.0, } for bad_account, guessed_version in even_worse_accounts.items(): req = Request.blank('/endpoints/%s/c/o' % bad_account) version, account, container, obj = \ self.list_endpoints._parse_path(req) self.assertEqual(version, guessed_version) self.assertEqual(account, 'c') self.assertEqual(container, 'o') self.assertEqual(obj, None)
def test_get_account_info_cache(self): # The original test that we prefer to preserve cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) resp = get_account_info(req.environ, FakeApp()) self.assertEqual(resp['bytes'], 3333) self.assertEqual(resp['total_object_count'], 10) self.assertEqual(resp['status'], 404) # Here is a more realistic test cached = {'status': 404, 'bytes': '3333', 'container_count': '234', 'total_object_count': '10', 'meta': {}} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) resp = get_account_info(req.environ, FakeApp()) self.assertEqual(resp['status'], 404) self.assertEqual(resp['bytes'], '3333') self.assertEqual(resp['container_count'], 234) self.assertEqual(resp['meta'], {}) self.assertEqual(resp['total_object_count'], '10')
def test_policy_index(self): # Policy index can be specified by X-Backend-Storage-Policy-Index # in the request header for object API app = proxy_logging.ProxyLoggingMiddleware(FakeApp(policy_idx="1"), {}) app.access_logger = FakeLogger() req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "PUT"}) resp = app(req.environ, start_response) "".join(resp) log_parts = self._log_parts(app) self.assertEqual(log_parts[20], "1") # Policy index can be specified by X-Backend-Storage-Policy-Index # in the response header for container API app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank("/v1/a/c", environ={"REQUEST_METHOD": "GET"}) def fake_call(app, env, start_response): start_response( app.response_str, [ ("Content-Type", "text/plain"), ("Content-Length", str(sum(map(len, app.body)))), ("X-Backend-Storage-Policy-Index", "1"), ], ) while env["wsgi.input"].read(5): pass return app.body with mock.patch.object(FakeApp, "__call__", fake_call): resp = app(req.environ, start_response) "".join(resp) log_parts = self._log_parts(app) self.assertEqual(log_parts[20], "1")
def create_container(self, req, container_path): """ Checks if the container exists and if not try to create it. :params container_path: an unquoted path to a container to be created :returns: True if created container, False if container exists :raises: CreateContainerError when unable to create container """ new_env = req.environ.copy() new_env['PATH_INFO'] = container_path new_env['swift.source'] = 'EA' new_env['REQUEST_METHOD'] = 'HEAD' head_cont_req = Request.blank(container_path, environ=new_env) resp = head_cont_req.get_response(self.app) if resp.is_success: return False if resp.status_int == 404: new_env = req.environ.copy() new_env['PATH_INFO'] = container_path new_env['swift.source'] = 'EA' new_env['REQUEST_METHOD'] = 'PUT' create_cont_req = Request.blank(container_path, environ=new_env) resp = create_cont_req.get_response(self.app) if resp.is_success: return True raise CreateContainerError( "Create Container Failed: " + container_path, resp.status_int, resp.status)
def test_PUT_encryption_override(self): # set crypto override to disable encryption. # simulate another middleware wanting to set footers other_footers = { "Etag": "other etag", "X-Object-Sysmeta-Other": "other sysmeta", "X-Object-Sysmeta-Container-Update-Override-Etag": "other override", } body = "FAKE APP" env = { "REQUEST_METHOD": "PUT", "swift.crypto.override": True, "swift.callback.update_footers": lambda footers: footers.update(other_footers), } hdrs = {"content-type": "text/plain", "content-length": str(len(body))} req = Request.blank("/v1/a/c/o", environ=env, body=body, headers=hdrs) self.app.register("PUT", "/v1/a/c/o", HTTPCreated, {}) resp = req.get_response(self.encrypter) self.assertEqual("201 Created", resp.status) # verify that other middleware's footers made it to app req_hdrs = self.app.headers[0] for k, v in other_footers.items(): self.assertEqual(v, req_hdrs[k]) # verify object is NOT encrypted by getting direct from the app get_req = Request.blank("/v1/a/c/o", environ={"REQUEST_METHOD": "GET"}) self.assertEqual(body, get_req.get_response(self.app).body)
def test_headers_to_sign_sigv4(self): environ = { 'REQUEST_METHOD': 'GET'} # host and x-amz-date x_amz_date = self.get_v4_amz_date_header() headers = { 'Authorization': 'AWS4-HMAC-SHA256 ' 'Credential=test/20130524/US/s3/aws4_request, ' 'SignedHeaders=host;x-amz-content-sha256;x-amz-date,' 'Signature=X', 'X-Amz-Content-SHA256': '0123456789', 'Date': self.get_date_header(), 'X-Amz-Date': x_amz_date} req = Request.blank('/', environ=environ, headers=headers) sigv4_req = SigV4Request(req.environ) headers_to_sign = sigv4_req._headers_to_sign() self.assertEqual(['host', 'x-amz-content-sha256', 'x-amz-date'], sorted(headers_to_sign.keys())) self.assertEqual(headers_to_sign['host'], 'localhost:80') self.assertEqual(headers_to_sign['x-amz-date'], x_amz_date) self.assertEqual(headers_to_sign['x-amz-content-sha256'], '0123456789') # no x-amz-date headers = { 'Authorization': 'AWS4-HMAC-SHA256 ' 'Credential=test/20130524/US/s3/aws4_request, ' 'SignedHeaders=host;x-amz-content-sha256,' 'Signature=X', 'X-Amz-Content-SHA256': '0123456789', 'Date': self.get_date_header()} req = Request.blank('/', environ=environ, headers=headers) sigv4_req = SigV4Request(req.environ) headers_to_sign = sigv4_req._headers_to_sign() self.assertEqual(['host', 'x-amz-content-sha256'], sorted(headers_to_sign.keys())) self.assertEqual(headers_to_sign['host'], 'localhost:80') self.assertEqual(headers_to_sign['x-amz-content-sha256'], '0123456789') # SignedHeaders says, host and x-amz-date included but there is not # X-Amz-Date header headers = { 'Authorization': 'AWS4-HMAC-SHA256 ' 'Credential=test/20130524/US/s3/aws4_request, ' 'SignedHeaders=host;x-amz-content-sha256;x-amz-date,' 'Signature=X', 'X-Amz-Content-SHA256': '0123456789', 'Date': self.get_date_header()} req = Request.blank('/', environ=environ, headers=headers) with self.assertRaises(SignatureDoesNotMatch): sigv4_req = SigV4Request(req.environ) sigv4_req._headers_to_sign()
def test_bucket_PUT(self): req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(body, '') self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Location'], '/bucket') # Apparently some clients will include a chunked transfer-encoding # even with no body req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'Transfer-Encoding': 'chunked'}) status, headers, body = self.call_s3api(req) self.assertEqual(body, '') self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Location'], '/bucket') with UnreadableInput(self) as fake_input: req = Request.blank( '/bucket', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': fake_input}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(body, '') self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Location'], '/bucket')
def test_sysmeta_replaced_by_PUT(self): path = '/v1/a/c/o' env = {'REQUEST_METHOD': 'PUT'} hdrs = dict(self.original_sysmeta_headers_1) hdrs.update(self.original_sysmeta_headers_2) hdrs.update(self.original_meta_headers_1) hdrs.update(self.original_meta_headers_2) req = Request.blank(path, environ=env, headers=hdrs, body='x') resp = req.get_response(self.app) self._assertStatus(resp, 201) env = {'REQUEST_METHOD': 'PUT'} hdrs = dict(self.changed_sysmeta_headers) hdrs.update(self.new_sysmeta_headers) hdrs.update(self.changed_meta_headers) hdrs.update(self.new_meta_headers) hdrs.update(self.bad_headers) req = Request.blank(path, environ=env, headers=hdrs, body='x') resp = req.get_response(self.app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) resp = req.get_response(self.app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.new_sysmeta_headers) self._assertNotInHeaders(resp, self.original_sysmeta_headers_2) self._assertInHeaders(resp, self.changed_meta_headers) self._assertInHeaders(resp, self.new_meta_headers) self._assertNotInHeaders(resp, self.original_meta_headers_2)
def test_upload_size(self): # Using default policy app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {"log_headers": "yes"}) app.access_logger = FakeLogger() req = Request.blank("/v1/a/c/o/foo", environ={"REQUEST_METHOD": "PUT", "wsgi.input": BytesIO(b"some stuff")}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEqual(log_parts[11], str(len("FAKE APP"))) self.assertEqual(log_parts[10], str(len("some stuff"))) self.assertUpdateStats( [ ("object.PUT.200.xfer", len("some stuff") + len("FAKE APP")), ("object.policy.0.PUT.200.xfer", len("some stuff") + len("FAKE APP")), ], app, ) # Using a non-existent policy app = proxy_logging.ProxyLoggingMiddleware(FakeApp(policy_idx="-1"), {"log_headers": "yes"}) app.access_logger = FakeLogger() req = Request.blank("/v1/a/c/o/foo", environ={"REQUEST_METHOD": "PUT", "wsgi.input": BytesIO(b"some stuff")}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEqual(log_parts[11], str(len("FAKE APP"))) self.assertEqual(log_parts[10], str(len("some stuff"))) self.assertUpdateStats([("object.PUT.200.xfer", len("some stuff") + len("FAKE APP"))], app)
def test_bucket_GET_v2_fetch_owner(self): bucket_name = 'junk' req = Request.blank('/%s?list-type=2' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') name = elem.find('./Name').text self.assertEqual(name, bucket_name) objects = elem.iterchildren('Contents') for o in objects: self.assertIsNone(o.find('./Owner')) req = Request.blank('/%s?list-type=2&fetch-owner=true' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') name = elem.find('./Name').text self.assertEqual(name, bucket_name) objects = elem.iterchildren('Contents') for o in objects: self.assertIsNotNone(o.find('./Owner'))
def test_bucket_GET_max_keys(self): class FakeApp(object): def __call__(self, env, start_response): self.query_string = env['QUERY_STRING'] start_response('200 OK', []) return '[]' fake_app = FakeApp() local_app = swift3.filter_factory({})(fake_app) bucket_name = 'junk' req = Request.blank('/%s' % bucket_name, environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING': 'max-keys=5'}, headers={'Authorization': 'AWS test:tester:hmac'}) resp = local_app(req.environ, lambda *args: None) dom = xml.dom.minidom.parseString("".join(resp)) self.assertEquals(dom.getElementsByTagName('MaxKeys')[0]. childNodes[0].nodeValue, '5') args = dict(cgi.parse_qsl(fake_app.query_string)) self.assert_(args['limit'] == '6') req = Request.blank('/%s' % bucket_name, environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING': 'max-keys=5000'}, headers={'Authorization': 'AWS test:tester:hmac'}) resp = local_app(req.environ, lambda *args: None) dom = xml.dom.minidom.parseString("".join(resp)) self.assertEquals(dom.getElementsByTagName('MaxKeys')[0]. childNodes[0].nodeValue, '1000') args = dict(cgi.parse_qsl(fake_app.query_string)) self.assertEquals(args['limit'], '1001')
def PUT(self, req): """HTTP PUT request handler.""" if req.if_none_match is not None and '*' not in req.if_none_match: # Sending an etag with if-none-match isn't currently supported return HTTPBadRequest(request=req, content_type='text/plain', body='If-None-Match only supports *') container_info = self.container_info( self.account_name, self.container_name, req) policy_index = req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy']) obj_ring = self.app.get_object_ring(policy_index) # pass the policy index to storage nodes via req header req.headers['X-Backend-Storage-Policy-Index'] = policy_index container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] object_versions = container_info['versions'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp if not containers: return HTTPNotFound(request=req) # Sometimes the 'content-type' header exists, but is set to None. content_type_manually_set = True detect_content_type = \ config_true_value(req.headers.get('x-detect-content-type')) if detect_content_type or not req.headers.get('content-type'): guessed_type, _junk = mimetypes.guess_type(req.path_info) req.headers['Content-Type'] = guessed_type or \ 'application/octet-stream' if detect_content_type: req.headers.pop('x-detect-content-type') else: content_type_manually_set = False error_response = check_object_creation(req, self.object_name) or \ check_content_type(req) if error_response: return error_response partition, nodes = obj_ring.get_nodes( self.account_name, self.container_name, self.object_name) # do a HEAD request for container sync and checking object versions if 'x-timestamp' in req.headers or \ (object_versions and not req.environ.get('swift_versioned_copy')): # make sure proxy-server uses the right policy index _headers = {'X-Backend-Storage-Policy-Index': policy_index, 'X-Newest': 'True'} hreq = Request.blank(req.path_info, headers=_headers, environ={'REQUEST_METHOD': 'HEAD'}) hresp = self.GETorHEAD_base( hreq, _('Object'), obj_ring, partition, hreq.swift_entity_path) # Used by container sync feature if 'x-timestamp' in req.headers: try: req_timestamp = Timestamp(req.headers['X-Timestamp']) if hresp.environ and 'swift_x_timestamp' in hresp.environ and \ hresp.environ['swift_x_timestamp'] >= req_timestamp: return HTTPAccepted(request=req) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) req.headers['X-Timestamp'] = req_timestamp.internal else: req.headers['X-Timestamp'] = Timestamp(time.time()).internal if object_versions and not req.environ.get('swift_versioned_copy'): if hresp.status_int != HTTP_NOT_FOUND: # This is a version manifest and needs to be handled # differently. First copy the existing data to a new object, # then write the data from this request to the version manifest # object. lcontainer = object_versions.split('/')[0] prefix_len = '%03x' % len(self.object_name) lprefix = prefix_len + self.object_name + '/' ts_source = hresp.environ.get('swift_x_timestamp') if ts_source is None: ts_source = time.mktime(time.strptime( hresp.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')) new_ts = Timestamp(ts_source).internal vers_obj_name = lprefix + new_ts copy_headers = { 'Destination': '%s/%s' % (lcontainer, vers_obj_name)} copy_environ = {'REQUEST_METHOD': 'COPY', 'swift_versioned_copy': True } copy_req = Request.blank(req.path_info, headers=copy_headers, environ=copy_environ) copy_resp = self.COPY(copy_req) if is_client_error(copy_resp.status_int): # missing container or bad permissions return HTTPPreconditionFailed(request=req) elif not is_success(copy_resp.status_int): # could not copy the data, bail return HTTPServiceUnavailable(request=req) reader = req.environ['wsgi.input'].read data_source = iter(lambda: reader(self.app.client_chunk_size), '') source_header = req.headers.get('X-Copy-From') source_resp = None if source_header: if req.environ.get('swift.orig_req_method', req.method) != 'POST': req.environ.setdefault('swift.log_info', []).append( 'x-copy-from:%s' % source_header) ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account', None) if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = check_copy_from_header(req) source_header = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) source_req = req.copy_get() # make sure the source request uses it's container_info source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.path_info = source_header source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name orig_container_name = self.container_name orig_account_name = self.account_name self.object_name = src_obj_name self.container_name = src_container_name self.account_name = src_account_name sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) source_resp = self.GET(source_req) # This gives middlewares a way to change the source; for example, # this lets you COPY a SLO manifest and have the new object be the # concatenation of the segments (like what a GET request gives # the client), not a copy of the manifest file. hook = req.environ.get( 'swift.copy_hook', (lambda source_req, source_resp, sink_req: source_resp)) source_resp = hook(source_req, source_resp, sink_req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: return source_resp self.object_name = orig_obj_name self.container_name = orig_container_name self.account_name = orig_account_name data_source = iter(source_resp.app_iter) sink_req.content_length = source_resp.content_length if sink_req.content_length is None: # This indicates a transfer-encoding: chunked source object, # which currently only happens because there are more than # CONTAINER_LISTING_LIMIT segments in a segmented object. In # this case, we're going to refuse to do the server-side copy. return HTTPRequestEntityTooLarge(request=req) if sink_req.content_length > constraints.MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) sink_req.etag = source_resp.etag # we no longer need the X-Copy-From header del sink_req.headers['X-Copy-From'] if 'X-Copy-From-Account' in sink_req.headers: del sink_req.headers['X-Copy-From-Account'] if not content_type_manually_set: sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] if config_true_value( sink_req.headers.get('x-fresh-metadata', 'false')): # post-as-copy: ignore new sysmeta, copy existing sysmeta condition = lambda k: is_sys_meta('object', k) remove_items(sink_req.headers, condition) copy_header_subset(source_resp, sink_req, condition) else: # copy/update existing sysmeta and user meta copy_headers_into(source_resp, sink_req) copy_headers_into(req, sink_req) # copy over x-static-large-object for POSTs and manifest copies if 'X-Static-Large-Object' in source_resp.headers and \ req.params.get('multipart-manifest') == 'get': sink_req.headers['X-Static-Large-Object'] = \ source_resp.headers['X-Static-Large-Object'] req = sink_req req, delete_at_container, delete_at_part, \ delete_at_nodes = self._config_obj_expiration(req) node_iter = GreenthreadSafeIterator( self.iter_nodes_local_first(obj_ring, partition)) pile = GreenPile(len(nodes)) te = req.headers.get('transfer-encoding', '') chunked = ('chunked' in te) outgoing_headers = self._backend_requests( req, len(nodes), container_partition, containers, delete_at_container, delete_at_part, delete_at_nodes) for nheaders in outgoing_headers: # RFC2616:8.2.3 disallows 100-continue without a body if (req.content_length > 0) or chunked: nheaders['Expect'] = '100-continue' pile.spawn(self._connect_put_node, node_iter, partition, req.swift_entity_path, nheaders, self.app.logger.thread_locals) conns = [conn for conn in pile if conn] min_conns = quorum_size(len(nodes)) if req.if_none_match is not None and '*' in req.if_none_match: statuses = [conn.resp.status for conn in conns if conn.resp] if HTTP_PRECONDITION_FAILED in statuses: # If we find any copy of the file, it shouldn't be uploaded self.app.logger.debug( _('Object PUT returning 412, %(statuses)r'), {'statuses': statuses}) return HTTPPreconditionFailed(request=req) if len(conns) < min_conns: self.app.logger.error( _('Object PUT returning 503, %(conns)s/%(nodes)s ' 'required connections'), {'conns': len(conns), 'nodes': min_conns}) return HTTPServiceUnavailable(request=req) bytes_transferred = 0 try: with ContextPool(len(nodes)) as pool: for conn in conns: conn.failed = False conn.queue = Queue(self.app.put_queue_depth) pool.spawn(self._send_file, conn, req.path) while True: with ChunkReadTimeout(self.app.client_timeout): try: chunk = next(data_source) except StopIteration: if chunked: for conn in conns: conn.queue.put('0\r\n\r\n') break bytes_transferred += len(chunk) if bytes_transferred > constraints.MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) for conn in list(conns): if not conn.failed: conn.queue.put( '%x\r\n%s\r\n' % (len(chunk), chunk) if chunked else chunk) else: conns.remove(conn) if len(conns) < min_conns: self.app.logger.error(_( 'Object PUT exceptions during' ' send, %(conns)s/%(nodes)s required connections'), {'conns': len(conns), 'nodes': min_conns}) return HTTPServiceUnavailable(request=req) for conn in conns: if conn.queue.unfinished_tasks: conn.queue.join() conns = [conn for conn in conns if not conn.failed] except ChunkReadTimeout as err: self.app.logger.warn( _('ERROR Client read timeout (%ss)'), err.seconds) self.app.logger.increment('client_timeouts') return HTTPRequestTimeout(request=req) except (Exception, Timeout): self.app.logger.exception( _('ERROR Exception causing client disconnect')) return HTTPClientDisconnect(request=req) if req.content_length and bytes_transferred < req.content_length: req.client_disconnect = True self.app.logger.warn( _('Client disconnected without sending enough data')) self.app.logger.increment('client_disconnects') return HTTPClientDisconnect(request=req) statuses, reasons, bodies, etags = self._get_put_responses(req, conns, nodes) if len(etags) > 1: self.app.logger.error( _('Object servers returned %s mismatched etags'), len(etags)) return HTTPServerError(request=req) etag = etags.pop() if len(etags) else None resp = self.best_response(req, statuses, reasons, bodies, _('Object PUT'), etag=etag) if source_header: acct, path = source_header.split('/', 3)[2:4] resp.headers['X-Copied-From-Account'] = quote(acct) resp.headers['X-Copied-From'] = quote(path) if 'last-modified' in source_resp.headers: resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) resp.last_modified = math.ceil( float(Timestamp(req.headers['X-Timestamp']))) return resp
def handle_PUT(self, req, start_response): if req.content_length: return HTTPBadRequest(body='Copy requests require a zero byte ' 'body', request=req, content_type='text/plain')(req.environ, start_response) # Form the path of source object to be fetched ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account') if src_account_name: src_account_name = check_account_format(req, unquote(src_account_name)) else: src_account_name = acct src_container_name, src_obj_name = _check_copy_from_header(req) source_path = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) # GET the source object, bail out on error ssc_ctx = ServerSideCopyWebContext(self.app, self.logger) source_resp = self._get_source_object(ssc_ctx, source_path, req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: return source_resp(source_resp.environ, start_response) # Create a new Request object based on the original request instance. # This will preserve original request environ including headers. sink_req = Request.blank(req.path_info, environ=req.environ) def is_object_sysmeta(k): return is_sys_meta('object', k) if config_true_value(req.headers.get('x-fresh-metadata', 'false')): # x-fresh-metadata only applies to copy, not post-as-copy: ignore # existing user metadata, update existing sysmeta with new copy_header_subset(source_resp, sink_req, is_object_sysmeta) copy_header_subset(req, sink_req, is_object_sysmeta) else: # First copy existing sysmeta, user meta and other headers from the # source to the sink, apart from headers that are conditionally # copied below and timestamps. exclude_headers = ('x-static-large-object', 'x-object-manifest', 'etag', 'content-type', 'x-timestamp', 'x-backend-timestamp') copy_header_subset(source_resp, sink_req, lambda k: k.lower() not in exclude_headers) # now update with original req headers sink_req.headers.update(req.headers) params = sink_req.params if params.get('multipart-manifest') == 'get': if 'X-Static-Large-Object' in source_resp.headers: params['multipart-manifest'] = 'put' if 'X-Object-Manifest' in source_resp.headers: del params['multipart-manifest'] sink_req.headers['X-Object-Manifest'] = \ source_resp.headers['X-Object-Manifest'] sink_req.params = params # Set swift.source, data source, content length and etag # for the PUT request sink_req.environ['swift.source'] = 'SSC' sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) sink_req.content_length = source_resp.content_length if (source_resp.status_int == HTTP_OK and 'X-Static-Large-Object' not in source_resp.headers and ('X-Object-Manifest' not in source_resp.headers or req.params.get('multipart-manifest') == 'get')): # copy source etag so that copied content is verified, unless: # - not a 200 OK response: source etag may not match the actual # content, for example with a 206 Partial Content response to a # ranged request # - SLO manifest: etag cannot be specified in manifest PUT; SLO # generates its own etag value which may differ from source # - SLO: etag in SLO response is not hash of actual content # - DLO: etag in DLO response is not hash of actual content sink_req.headers['Etag'] = source_resp.etag else: # since we're not copying the source etag, make sure that any # container update override values are not copied. remove_items( sink_req.headers, lambda k: k.startswith( 'X-Object-Sysmeta-Container-Update-Override-')) # We no longer need these headers sink_req.headers.pop('X-Copy-From', None) sink_req.headers.pop('X-Copy-From-Account', None) # If the copy request does not explicitly override content-type, # use the one present in the source object. if not req.headers.get('content-type'): sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] # Create response headers for PUT response resp_headers = self._create_response_headers(source_path, source_resp, sink_req) put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response) close_if_possible(source_resp.app_iter) return put_resp
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: if req.method not in self.allowed_methods: raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: method = getattr(self, req.method) res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s' ' %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, '') if req.method in ('REPLICATE', 'SSYNC') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) # To be able to zero-copy send the object, we need a few things. # First, we have to be responding successfully to a GET, or else we're # not sending the object. Second, we have to be able to extract the # socket file descriptor from the WSGI input object. Third, the # diskfile has to support zero-copy send. # # There's a good chance that this could work for 206 responses too, # but the common case is sending the whole object, so we'll start # there. if req.method == 'GET' and res.status_int == 200 and \ isinstance(env['wsgi.input'], wsgi.Input): app_iter = getattr(res, 'app_iter', None) checker = getattr(app_iter, 'can_zero_copy_send', None) if checker and checker(): # For any kind of zero-copy thing like sendfile or splice, we # need the file descriptor. Eventlet doesn't provide a clean # way of getting that, so we resort to this. wsock = env['wsgi.input'].get_socket() wsockfd = wsock.fileno() # Don't call zero_copy_send() until after we force the HTTP # headers out of Eventlet and into the socket. def zero_copy_iter(): # If possible, set TCP_CORK so that headers don't # immediately go on the wire, but instead, wait for some # response body to make the TCP frames as large as # possible (and hence as few packets as possible). # # On non-Linux systems, we might consider TCP_NODELAY, but # since the only known zero-copy-capable diskfile uses # Linux-specific syscalls, we'll defer that work until # someone needs it. if hasattr(socket, 'TCP_CORK'): wsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1) yield EventletPlungerString() try: app_iter.zero_copy_send(wsockfd) except Exception: self.logger.exception("zero_copy_send() blew up") raise yield '' # Get headers ready to go out res(env, start_response) return zero_copy_iter() else: return res(env, start_response) else: return res(env, start_response)
def __call__(self, env, start_response): start_response_args = [None] input_proxy = InputProxy(env['wsgi.input']) env['wsgi.input'] = input_proxy start_time = time.time() def my_start_response(status, headers, exc_info=None): start_response_args[0] = (status, list(headers), exc_info) def status_int_for_logging(client_disconnect=False, start_status=None): # log disconnected clients as '499' status code if client_disconnect or input_proxy.client_disconnect: ret_status_int = 499 elif start_status is None: ret_status_int = int( start_response_args[0][0].split(' ', 1)[0]) else: ret_status_int = start_status return ret_status_int def iter_response(iterable): iterator = iter(iterable) try: chunk = iterator.next() while not chunk: chunk = iterator.next() except StopIteration: chunk = '' for h, v in start_response_args[0][1]: if h.lower() in ('content-length', 'transfer-encoding'): break else: if not chunk: start_response_args[0][1].append(('content-length', '0')) elif isinstance(iterable, list): start_response_args[0][1].append( ('content-length', str(sum(len(i) for i in iterable)))) start_response(*start_response_args[0]) req = Request(env) # Log timing information for time-to-first-byte (GET requests only) method = self.method_from_req(req) if method == 'GET' and not self.req_already_logged(req): status_int = status_int_for_logging() metric_name = self.statsd_metric_name(req, status_int, method) if metric_name: self.access_logger.timing_since( metric_name + '.first-byte.timing', start_time) bytes_sent = 0 client_disconnect = False try: while chunk: bytes_sent += len(chunk) yield chunk chunk = iterator.next() except GeneratorExit: # generator was closed before we finished client_disconnect = True raise finally: status_int = status_int_for_logging(client_disconnect) self.log_request( req, status_int, input_proxy.bytes_received, bytes_sent, start_time, time.time()) try: iterable = self.app(env, my_start_response) except Exception: req = Request(env) status_int = status_int_for_logging(start_status=500) self.log_request( req, status_int, input_proxy.bytes_received, 0, start_time, time.time()) raise else: return iter_response(iterable)
def test_cache_middleware(self): req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertTrue('swift.cache' in resp) self.assertTrue(isinstance(resp['swift.cache'], MemcacheRing))
def __call__(self, env, start_response): req = Request(env) if req.path.startswith('/recon/'): return self.GET(req)(env, start_response) else: return self.app(env, start_response)
def DELETE(self, req): """HTTP DELETE request handler.""" container_info = self.container_info( self.account_name, self.container_name, req) # pass the policy index to storage nodes via req header policy_index = req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy']) obj_ring = self.app.get_object_ring(policy_index) # pass the policy index to storage nodes via req header req.headers['X-Backend-Storage-Policy-Index'] = policy_index container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] object_versions = container_info['versions'] if object_versions: # this is a version manifest and needs to be handled differently object_versions = unquote(object_versions) lcontainer = object_versions.split('/')[0] prefix_len = '%03x' % len(self.object_name) lprefix = prefix_len + self.object_name + '/' item_list = [] try: for _item in self._listing_iter(lcontainer, lprefix, req.environ): item_list.append(_item) except ListingIterNotFound: # no worries, last_item is None pass except ListingIterNotAuthorized as err: return err.aresp except ListingIterError: return HTTPServerError(request=req) while len(item_list) > 0: previous_version = item_list.pop() # there are older versions so copy the previous version to the # current object and delete the previous version orig_container = self.container_name orig_obj = self.object_name self.container_name = lcontainer self.object_name = previous_version['name'].encode('utf-8') copy_path = '/v1/' + self.account_name + '/' + \ self.container_name + '/' + self.object_name copy_headers = {'X-Newest': 'True', 'Destination': orig_container + '/' + orig_obj } copy_environ = {'REQUEST_METHOD': 'COPY', 'swift_versioned_copy': True } creq = Request.blank(copy_path, headers=copy_headers, environ=copy_environ) copy_resp = self.COPY(creq) if copy_resp.status_int == HTTP_NOT_FOUND: # the version isn't there so we'll try with previous self.container_name = orig_container self.object_name = orig_obj continue if is_client_error(copy_resp.status_int): # some user error, maybe permissions return HTTPPreconditionFailed(request=req) elif not is_success(copy_resp.status_int): # could not copy the data, bail return HTTPServiceUnavailable(request=req) # reset these because the COPY changed them self.container_name = lcontainer self.object_name = previous_version['name'].encode('utf-8') new_del_req = Request.blank(copy_path, environ=req.environ) container_info = self.container_info( self.account_name, self.container_name, req) policy_idx = container_info['storage_policy'] obj_ring = self.app.get_object_ring(policy_idx) # pass the policy index to storage nodes via req header new_del_req.headers['X-Backend-Storage-Policy-Index'] = \ policy_idx container_partition = container_info['partition'] containers = container_info['nodes'] new_del_req.acl = container_info['write_acl'] new_del_req.path_info = copy_path req = new_del_req # remove 'X-If-Delete-At', since it is not for the older copy if 'X-If-Delete-At' in req.headers: del req.headers['X-If-Delete-At'] break if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp if not containers: return HTTPNotFound(request=req) partition, nodes = obj_ring.get_nodes( self.account_name, self.container_name, self.object_name) # Used by container sync feature if 'x-timestamp' in req.headers: try: req_timestamp = Timestamp(req.headers['X-Timestamp']) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) req.headers['X-Timestamp'] = req_timestamp.internal else: req.headers['X-Timestamp'] = Timestamp(time.time()).internal headers = self._backend_requests( req, len(nodes), container_partition, containers) # When deleting objects treat a 404 status as 204. status_overrides = {404: 204} resp = self.make_requests(req, obj_ring, partition, 'DELETE', req.swift_entity_path, headers, overrides=status_overrides) return resp
def test_object_multi_DELETE_with_error(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3', swob.HTTPForbidden, {}, None) self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key4', swob.HTTPOk, {'x-static-large-object': 'True'}, None) slo_delete_resp = { 'Number Not Found': 0, 'Response Status': '400 Bad Request', 'Errors': [["/bucket+segments/obj1", "403 Forbidden"], ["/bucket+segments/obj2", "403 Forbidden"]], 'Response Body': '', 'Number Deleted': 8 } self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key4', swob.HTTPOk, {}, json.dumps(slo_delete_resp)) elem = Element('Delete') for key in ['Key1', 'Key2', 'Key3', 'Key4']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Content-Type': 'multipart/form-data', 'Date': self.get_date_header(), 'Content-MD5': content_md5 }, body=body) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body) self.assertEqual(len(elem.findall('Deleted')), 2) self.assertEqual(len(elem.findall('Error')), 2) self.assertEqual( [(el.find('Code').text, el.find('Message').text) for el in elem.findall('Error')], [('AccessDenied', 'Access Denied.'), ('SLODeleteError', '\n'.join([ '400 Bad Request', '/bucket+segments/obj1: 403 Forbidden', '/bucket+segments/obj2: 403 Forbidden' ]))]) self.assertEqual(self.swift.calls, [ ('HEAD', '/v1/AUTH_test/bucket'), ('HEAD', '/v1/AUTH_test/bucket/Key1'), ('DELETE', '/v1/AUTH_test/bucket/Key1'), ('HEAD', '/v1/AUTH_test/bucket/Key2'), ('HEAD', '/v1/AUTH_test/bucket/Key3'), ('HEAD', '/v1/AUTH_test/bucket/Key4'), ('DELETE', '/v1/AUTH_test/bucket/Key4?multipart-manifest=delete'), ])
def test_check_metadata_good(self): headers = {'X-Object-Meta-Name': 'Value'} self.assertIsNone( constraints.check_metadata(Request.blank('/', headers=headers), 'object'))
def test_get_endpoint(self): # Expected results for objects taken from test_ring # Expected results for others computed by manually invoking # ring.get_nodes(). resp = Request.blank('/endpoints/a/c/o1').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/1/a/c/o1", "http://10.1.2.2:6000/sdd1/1/a/c/o1" ]) # Here, 'o1/' is the object name. resp = Request.blank('/endpoints/a/c/o1/').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/3/a/c/o1/", "http://10.1.2.2:6000/sdd1/3/a/c/o1/" ]) resp = Request.blank('/endpoints/a/c2').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sda1/2/a/c2", "http://10.1.2.1:6000/sdc1/2/a/c2" ]) resp = Request.blank('/endpoints/a1').get_response(self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.2.1:6000/sdc1/0/a1", "http://10.1.1.1:6000/sda1/0/a1", "http://10.1.1.1:6000/sdb1/0/a1" ]) resp = Request.blank('/endpoints/').get_response(self.list_endpoints) self.assertEquals(resp.status_int, 400) resp = Request.blank('/endpoints/a/c 2').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/3/a/c%202", "http://10.1.2.2:6000/sdd1/3/a/c%202" ]) resp = Request.blank('/endpoints/a/c%202').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/3/a/c%202", "http://10.1.2.2:6000/sdd1/3/a/c%202" ]) resp = Request.blank('/endpoints/ac%20count/con%20tainer/ob%20ject') \ .get_response(self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/3/ac%20count/con%20tainer/ob%20ject", "http://10.1.2.2:6000/sdd1/3/ac%20count/con%20tainer/ob%20ject" ]) resp = Request.blank('/endpoints/a/c/o1', {'REQUEST_METHOD': 'POST'}) \ .get_response(self.list_endpoints) self.assertEquals(resp.status_int, 405) self.assertEquals(resp.status, '405 Method Not Allowed') self.assertEquals(resp.headers['allow'], 'GET') resp = Request.blank('/not-endpoints').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.status, '200 OK') self.assertEquals(resp.body, 'FakeApp') # test custom path with trailing slash custom_path_le = list_endpoints.filter_factory({ 'swift_dir': self.testdir, 'list_endpoints_path': '/some/another/path/' })(self.app) resp = Request.blank('/some/another/path/a/c/o1') \ .get_response(custom_path_le) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/1/a/c/o1", "http://10.1.2.2:6000/sdd1/1/a/c/o1" ]) # test ustom path without trailing slash custom_path_le = list_endpoints.filter_factory({ 'swift_dir': self.testdir, 'list_endpoints_path': '/some/another/path' })(self.app) resp = Request.blank('/some/another/path/a/c/o1') \ .get_response(custom_path_le) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/1/a/c/o1", "http://10.1.2.2:6000/sdd1/1/a/c/o1" ])
def test_check_delete_headers(self): # X-Delete-After headers = {'X-Delete-After': '60'} resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) self.assertTrue(isinstance(resp, Request)) self.assertTrue('x-delete-at' in resp.headers) headers = {'X-Delete-After': 'abc'} try: resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) except HTTPException as e: self.assertEqual(e.status_int, HTTP_BAD_REQUEST) self.assertTrue('Non-integer X-Delete-After' in e.body) else: self.fail("Should have failed with HTTPBadRequest") headers = {'X-Delete-After': '60.1'} try: resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) except HTTPException as e: self.assertEqual(e.status_int, HTTP_BAD_REQUEST) self.assertTrue('Non-integer X-Delete-After' in e.body) else: self.fail("Should have failed with HTTPBadRequest") headers = {'X-Delete-After': '-1'} try: resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) except HTTPException as e: self.assertEqual(e.status_int, HTTP_BAD_REQUEST) self.assertTrue('X-Delete-After in past' in e.body) else: self.fail("Should have failed with HTTPBadRequest") # X-Delete-At t = str(int(time.time() + 100)) headers = {'X-Delete-At': t} resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) self.assertTrue(isinstance(resp, Request)) self.assertTrue('x-delete-at' in resp.headers) self.assertEqual(resp.headers.get('X-Delete-At'), t) headers = {'X-Delete-At': 'abc'} try: resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) except HTTPException as e: self.assertEqual(e.status_int, HTTP_BAD_REQUEST) self.assertTrue('Non-integer X-Delete-At' in e.body) else: self.fail("Should have failed with HTTPBadRequest") t = str(int(time.time() + 100)) + '.1' headers = {'X-Delete-At': t} try: resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) except HTTPException as e: self.assertEqual(e.status_int, HTTP_BAD_REQUEST) self.assertTrue('Non-integer X-Delete-At' in e.body) else: self.fail("Should have failed with HTTPBadRequest") t = str(int(time.time())) headers = {'X-Delete-At': t} try: resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) except HTTPException as e: self.assertEqual(e.status_int, HTTP_BAD_REQUEST) self.assertTrue('X-Delete-At in past' in e.body) else: self.fail("Should have failed with HTTPBadRequest") t = str(int(time.time() - 1)) headers = {'X-Delete-At': t} try: resp = constraints.check_delete_headers( Request.blank('/', headers=headers)) except HTTPException as e: self.assertEqual(e.status_int, HTTP_BAD_REQUEST) self.assertTrue('X-Delete-At in past' in e.body) else: self.fail("Should have failed with HTTPBadRequest")
def test_check_metadata_empty_name(self): headers = {'X-Object-Meta-': 'Value'} resp = constraints.check_metadata(Request.blank('/', headers=headers), 'object') self.assertEqual(resp.status_int, HTTP_BAD_REQUEST) self.assertIn('Metadata name cannot be empty', resp.body)
def handle_multipart_put(self, req, start_response): """ Will handle the PUT of a SLO manifest. Heads every object in manifest to check if is valid and if so will save a manifest generated from the user input. Uses WSGIContext to call self and start_response and returns a WSGI iterator. :params req: a swob.Request with an obj in path :raises: HttpException on errors """ try: vrs, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(req.environ, start_response) if req.content_length > self.max_manifest_size: raise HTTPRequestEntityTooLarge("Manifest File > %d bytes" % self.max_manifest_size) if req.headers.get('X-Copy-From'): raise HTTPMethodNotAllowed( 'Multipart Manifest PUTs cannot be COPY requests') if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) parsed_data = parse_input(req.body_file.read(self.max_manifest_size)) problem_segments = [] if len(parsed_data) > self.max_manifest_segments: raise HTTPRequestEntityTooLarge( 'Number of segments must be <= %d' % self.max_manifest_segments) total_size = 0 out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if not out_content_type: out_content_type = 'text/plain' data_for_storage = [] slo_etag = md5() for index, seg_dict in enumerate(parsed_data): obj_name = seg_dict['path'] if isinstance(obj_name, unicode): obj_name = obj_name.encode('utf-8') obj_path = '/'.join(['', vrs, account, obj_name.lstrip('/')]) if req.path == quote(obj_path): raise HTTPConflict('Manifest object name "%s" ' 'cannot be included in the manifest' % obj_name) try: seg_size = int(seg_dict['size_bytes']) except (ValueError, TypeError): raise HTTPBadRequest('Invalid Manifest File') if seg_size < self.min_segment_size and \ index < len(parsed_data) - 1: raise HTTPBadRequest( 'Each segment, except the last, must be at least ' '%d bytes.' % self.min_segment_size) new_env = req.environ.copy() new_env['PATH_INFO'] = obj_path new_env['REQUEST_METHOD'] = 'HEAD' new_env['swift.source'] = 'SLO' del (new_env['wsgi.input']) del (new_env['QUERY_STRING']) new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT') head_seg_resp = \ Request.blank(obj_path, new_env).get_response(self) if head_seg_resp.is_success: total_size += seg_size if seg_size != head_seg_resp.content_length: problem_segments.append([quote(obj_name), 'Size Mismatch']) if seg_dict['etag'] == head_seg_resp.etag: slo_etag.update(seg_dict['etag']) else: problem_segments.append([quote(obj_name), 'Etag Mismatch']) if head_seg_resp.last_modified: last_modified = head_seg_resp.last_modified else: # shouldn't happen last_modified = datetime.now() last_modified_formatted = \ last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f') seg_data = { 'name': '/' + seg_dict['path'].lstrip('/'), 'bytes': seg_size, 'hash': seg_dict['etag'], 'content_type': head_seg_resp.content_type, 'last_modified': last_modified_formatted } if config_true_value( head_seg_resp.headers.get('X-Static-Large-Object')): seg_data['sub_slo'] = True data_for_storage.append(seg_data) else: problem_segments.append( [quote(obj_name), head_seg_resp.status]) if problem_segments: resp_body = get_response_body(out_content_type, {}, problem_segments) raise HTTPBadRequest(resp_body, content_type=out_content_type) env = req.environ if not env.get('CONTENT_TYPE'): guessed_type, _junk = mimetypes.guess_type(req.path_info) env['CONTENT_TYPE'] = guessed_type or 'application/octet-stream' env['swift.content_type_overridden'] = True env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True' json_data = json.dumps(data_for_storage) env['CONTENT_LENGTH'] = str(len(json_data)) env['wsgi.input'] = StringIO(json_data) slo_put_context = SloPutContext(self, slo_etag) return slo_put_context.handle_slo_put(req, start_response)
def test_check_metadata_empty(self): headers = {} self.assertIsNone( constraints.check_metadata(Request.blank('/', headers=headers), 'object'))
def test_GET_not_found(self): req = Request.blank('/v1/a/c/o') self.storage.object_fetch = Mock(side_effect=exc.NoSuchObject) resp = req.get_response(self.app) self.assertEqual(resp.status_int, 404)
def test_check_object_creation_bad_content_type(self): headers = {'Transfer-Encoding': 'chunked', 'Content-Type': '\xff\xff'} resp = constraints.check_object_creation( Request.blank('/', headers=headers), 'object_name') self.assertEqual(resp.status_int, HTTP_BAD_REQUEST) self.assertTrue('Content-Type' in resp.body)
def test_PUT_requires_length(self): req = Request.blank('/v1/a/c/o', method='PUT') resp = req.get_response(self.app) self.assertEqual(resp.status_int, 411)
def make_request(self, method, path, headers, acceptable_statuses, body_file=None, params=None): """Makes a request to Swift with retries. :param method: HTTP method of request. :param path: Path of request. :param headers: Headers to be sent with request. :param acceptable_statuses: List of acceptable statuses for request. :param body_file: Body file to be passed along with request, defaults to None. :param params: A dict of params to be set in request query string, defaults to None. :returns: Response object on success. :raises UnexpectedResponse: Exception raised when make_request() fails to get a response with an acceptable status :raises Exception: Exception is raised when code fails in an unexpected way. """ headers = dict(headers) headers['user-agent'] = self.user_agent for attempt in range(self.request_tries): resp = exc_type = exc_value = exc_traceback = None req = Request.blank(path, environ={'REQUEST_METHOD': method}, headers=headers) if body_file is not None: if hasattr(body_file, 'seek'): body_file.seek(0) req.body_file = body_file if params: req.params = params try: resp = req.get_response(self.app) except (Exception, Timeout): exc_type, exc_value, exc_traceback = exc_info() else: if resp.status_int in acceptable_statuses or \ resp.status_int // 100 in acceptable_statuses: return resp elif not is_server_error(resp.status_int): # No sense retrying when we expect the same result break # sleep only between tries, not after each one if attempt < self.request_tries - 1: if resp: # always close any resp.app_iter before we discard it with closing_if_possible(resp.app_iter): # for non 2XX requests it's safe and useful to drain # the response body so we log the correct status code if resp.status_int // 100 != 2: for iter_body in resp.app_iter: pass sleep(2**(attempt + 1)) if resp: msg = 'Unexpected response: %s' % resp.status if resp.status_int // 100 != 2 and resp.body: # provide additional context (and drain the response body) for # non 2XX responses msg += ' (%s)' % resp.body raise UnexpectedResponse(msg, resp) if exc_type: # To make pep8 tool happy, in place of raise t, v, tb: six.reraise(exc_type, exc_value, exc_traceback)
def test_PUT_req(self): body_key = os.urandom(32) object_key = fetch_crypto_keys()['object'] plaintext = b'FAKE APP' plaintext_etag = md5hex(plaintext) ciphertext = encrypt(plaintext, body_key, FAKE_IV) ciphertext_etag = md5hex(ciphertext) env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} hdrs = { 'etag': plaintext_etag, 'content-type': 'text/plain', 'content-length': str(len(plaintext)), 'x-object-meta-etag': 'not to be confused with the Etag!', 'x-object-meta-test': 'encrypt me', 'x-object-sysmeta-test': 'do not encrypt me' } req = Request.blank('/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) with mock.patch( 'swift.common.middleware.crypto.crypto_utils.' 'Crypto.create_random_key', return_value=body_key): resp = req.get_response(self.encrypter) self.assertEqual('201 Created', resp.status) self.assertEqual(plaintext_etag, resp.headers['Etag']) # verify metadata items self.assertEqual(1, len(self.app.calls), self.app.calls) self.assertEqual('PUT', self.app.calls[0][0]) req_hdrs = self.app.headers[0] # verify body crypto meta actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] actual = json.loads(urlparse.unquote_plus(actual)) self.assertEqual(Crypto().cipher, actual['cipher']) self.assertEqual(FAKE_IV, base64.b64decode(actual['iv'])) # verify wrapped body key expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) self.assertEqual(expected_wrapped_key, base64.b64decode(actual['body_key']['key'])) self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv'])) self.assertEqual(fetch_crypto_keys()['id'], actual['key_id']) # verify etag self.assertEqual(ciphertext_etag, req_hdrs['Etag']) encrypted_etag, _junk, etag_meta = \ req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=') # verify crypto_meta was appended to this etag self.assertTrue(etag_meta) actual_meta = json.loads(urlparse.unquote_plus(etag_meta)) self.assertEqual(Crypto().cipher, actual_meta['cipher']) # verify encrypted version of plaintext etag actual = base64.b64decode(encrypted_etag) etag_iv = base64.b64decode(actual_meta['iv']) enc_etag = encrypt(plaintext_etag.encode('ascii'), object_key, etag_iv) self.assertEqual(enc_etag, actual) # verify etag MAC for conditional requests actual_hmac = base64.b64decode( req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac']) exp_hmac = hmac.new(object_key, plaintext_etag.encode('ascii'), hashlib.sha256).digest() self.assertEqual(actual_hmac, exp_hmac) # verify encrypted etag for container update self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) parts = req_hdrs[ 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) self.assertEqual(2, len(parts)) # extract crypto_meta from end of etag for container update param = parts[1].strip() crypto_meta_tag = 'swift_meta=' self.assertTrue(param.startswith(crypto_meta_tag), param) actual_meta = json.loads( urlparse.unquote_plus(param[len(crypto_meta_tag):])) self.assertEqual(Crypto().cipher, actual_meta['cipher']) self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id']) cont_key = fetch_crypto_keys()['container'] cont_etag_iv = base64.b64decode(actual_meta['iv']) self.assertEqual(FAKE_IV, cont_etag_iv) exp_etag = encrypt(plaintext_etag.encode('ascii'), cont_key, cont_etag_iv) self.assertEqual(exp_etag, base64.b64decode(parts[0])) # content-type is not encrypted self.assertEqual('text/plain', req_hdrs['Content-Type']) # user meta is encrypted self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key) self._verify_user_metadata(req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key) # sysmeta is not encrypted self.assertEqual('do not encrypt me', req_hdrs['X-Object-Sysmeta-Test']) # verify object is encrypted by getting direct from the app get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = get_req.get_response(self.app) self.assertEqual(ciphertext, resp.body) self.assertEqual(ciphertext_etag, resp.headers['Etag'])
def test_PUT_if_none_match_not_star(self): req = Request.blank('/v1/a/c/o', method='PUT') req.headers['if-none-match'] = 'foo' req.headers['content-length'] = '0' resp = req.get_response(self.app) self.assertEqual(resp.status_int, 400)
def _test_PUT_with_other_footers(self, override_etag): # verify handling of another middleware's footer callback body_key = os.urandom(32) object_key = fetch_crypto_keys()['object'] plaintext = b'FAKE APP' plaintext_etag = md5hex(plaintext) ciphertext = encrypt(plaintext, body_key, FAKE_IV) ciphertext_etag = md5hex(ciphertext) other_footers = { 'Etag': plaintext_etag, 'X-Object-Sysmeta-Other': 'other sysmeta', 'X-Object-Sysmeta-Container-Update-Override-Size': 'other override', 'X-Object-Sysmeta-Container-Update-Override-Etag': override_etag } env = { 'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys, 'swift.callback.update_footers': lambda footers: footers.update(other_footers) } hdrs = { 'content-type': 'text/plain', 'content-length': str(len(plaintext)), 'Etag': 'correct etag is in footers' } req = Request.blank('/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) with mock.patch( 'swift.common.middleware.crypto.crypto_utils.' 'Crypto.create_random_key', lambda *args: body_key): resp = req.get_response(self.encrypter) self.assertEqual('201 Created', resp.status) self.assertEqual(plaintext_etag, resp.headers['Etag']) # verify metadata items self.assertEqual(1, len(self.app.calls), self.app.calls) self.assertEqual('PUT', self.app.calls[0][0]) req_hdrs = self.app.headers[0] # verify that other middleware's footers made it to app, including any # container update overrides but nothing Etag-related other_footers.pop('Etag') other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag') for k, v in other_footers.items(): self.assertEqual(v, req_hdrs[k]) # verify encryption footers are ok encrypted_etag, _junk, etag_meta = \ req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=') self.assertTrue(etag_meta) actual_meta = json.loads(urlparse.unquote_plus(etag_meta)) self.assertEqual(Crypto().cipher, actual_meta['cipher']) self.assertEqual(ciphertext_etag, req_hdrs['Etag']) actual = base64.b64decode(encrypted_etag) etag_iv = base64.b64decode(actual_meta['iv']) exp_etag = encrypt(plaintext_etag.encode('ascii'), object_key, etag_iv) self.assertEqual(exp_etag, actual) # verify encrypted etag for container update self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) parts = req_hdrs[ 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) self.assertEqual(2, len(parts)) # extract crypto_meta from end of etag for container update param = parts[1].strip() crypto_meta_tag = 'swift_meta=' self.assertTrue(param.startswith(crypto_meta_tag), param) actual_meta = json.loads( urlparse.unquote_plus(param[len(crypto_meta_tag):])) self.assertEqual(Crypto().cipher, actual_meta['cipher']) cont_key = fetch_crypto_keys()['container'] cont_etag_iv = base64.b64decode(actual_meta['iv']) self.assertEqual(FAKE_IV, cont_etag_iv) exp_etag = encrypt(override_etag.encode('ascii'), cont_key, cont_etag_iv) self.assertEqual(exp_etag, base64.b64decode(parts[0])) # verify body crypto meta actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] actual = json.loads(urlparse.unquote_plus(actual)) self.assertEqual(Crypto().cipher, actual['cipher']) self.assertEqual(FAKE_IV, base64.b64decode(actual['iv'])) # verify wrapped body key expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) self.assertEqual(expected_wrapped_key, base64.b64decode(actual['body_key']['key'])) self.assertEqual(FAKE_IV, base64.b64decode(actual['body_key']['iv'])) self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
def _make_request(self, path, **kwargs): req = Request.blank(path, **kwargs) req.environ['swift.cache'] = FakeMemcache() return req
def do_test(host): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': host}) return app(req.environ, start_response)
def test_PUT_nothing_read(self): # simulate an artificial scenario of a downstream filter/app not # actually reading the input stream from encrypter. class NonReadingApp(object): def __call__(self, env, start_response): # note: no read from wsgi.input req = Request(env) env['swift.callback.update_footers'](req.headers) call_headers.append(req.headers) resp = HTTPCreated(req=req, headers={'Etag': 'response etag'}) return resp(env, start_response) env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} hdrs = { 'content-type': 'text/plain', 'content-length': 0, 'etag': 'etag from client' } req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) call_headers = [] resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) self.assertEqual('201 Created', resp.status) self.assertEqual('response etag', resp.headers['Etag']) self.assertEqual(1, len(call_headers)) self.assertEqual('etag from client', call_headers[0]['etag']) # verify no encryption footers for k in call_headers[0]: self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-')) # check that an upstream footer callback gets called other_footers = { 'Etag': EMPTY_ETAG, 'X-Object-Sysmeta-Other': 'other sysmeta', 'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override' } env.update({ 'swift.callback.update_footers': lambda footers: footers.update(other_footers) }) req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) call_headers = [] resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) self.assertEqual('201 Created', resp.status) self.assertEqual('response etag', resp.headers['Etag']) self.assertEqual(1, len(call_headers)) # verify encrypted override etag for container update. self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', call_headers[0]) parts = call_headers[0][ 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) self.assertEqual(2, len(parts)) cont_key = fetch_crypto_keys()['container'] param = parts[1].strip() crypto_meta_tag = 'swift_meta=' self.assertTrue(param.startswith(crypto_meta_tag), param) actual_meta = json.loads( urlparse.unquote_plus(param[len(crypto_meta_tag):])) self.assertEqual(Crypto().cipher, actual_meta['cipher']) self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id']) cont_etag_iv = base64.b64decode(actual_meta['iv']) self.assertEqual(FAKE_IV, cont_etag_iv) self.assertEqual(encrypt(b'other override', cont_key, cont_etag_iv), base64.b64decode(parts[0])) # verify that other middleware's footers made it to app other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag') for k, v in other_footers.items(): self.assertEqual(v, call_headers[0][k]) # verify no encryption footers for k in call_headers[0]: self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-')) # if upstream footer override etag is for an empty body then check that # it is not encrypted other_footers = { 'Etag': EMPTY_ETAG, 'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG } env.update({ 'swift.callback.update_footers': lambda footers: footers.update(other_footers) }) req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) call_headers = [] resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) self.assertEqual('201 Created', resp.status) self.assertEqual('response etag', resp.headers['Etag']) self.assertEqual(1, len(call_headers)) # verify that other middleware's footers made it to app for k, v in other_footers.items(): self.assertEqual(v, call_headers[0][k]) # verify no encryption footers for k in call_headers[0]: self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-')) # if upstream footer override etag is an empty string then check that # it is not encrypted other_footers = { 'Etag': EMPTY_ETAG, 'X-Object-Sysmeta-Container-Update-Override-Etag': '' } env.update({ 'swift.callback.update_footers': lambda footers: footers.update(other_footers) }) req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) call_headers = [] resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) self.assertEqual('201 Created', resp.status) self.assertEqual('response etag', resp.headers['Etag']) self.assertEqual(1, len(call_headers)) # verify that other middleware's footers made it to app for k, v in other_footers.items(): self.assertEqual(v, call_headers[0][k]) # verify no encryption footers for k in call_headers[0]: self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
def test_cname_matching_ending_not_domain(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'foo.com'}) resp = self.app(req.environ, start_response) self.assertEqual(resp, ['CNAME lookup failed to resolve to a valid domain'])
def test_PUT_app_exception(self): app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {}) req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'}) with self.assertRaises(HTTPException) as catcher: req.get_response(app) self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body)
def test_something_weird(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'mysite.com'}) resp = self.app(req.environ, start_response) self.assertEqual(resp, ['CNAME lookup failed to resolve to a valid domain'])
def do_test(lookup_back): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.a.example.com'}) module = 'swift.common.middleware.cname_lookup.lookup_cname' with mock.patch(module, lambda d, r: (0, lookup_back)): return app(req.environ, start_response)
def test_object_HEAD_error(self): # HEAD does not return the body even an error response in the # specifications of the REST API. # So, check the response code for error test of HEAD. req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'HEAD'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPUnauthorized, {}, None) status, headers, body = self.call_swift3(req) self.assertEqual(status.split()[0], '403') self.assertEqual(body, '') # sanity req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'HEAD'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPForbidden, {}, None) status, headers, body = self.call_swift3(req) self.assertEqual(status.split()[0], '403') self.assertEqual(body, '') # sanity req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'HEAD'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPNotFound, {}, None) status, headers, body = self.call_swift3(req) self.assertEqual(status.split()[0], '404') self.assertEqual(body, '') # sanity req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'HEAD'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPPreconditionFailed, {}, None) status, headers, body = self.call_swift3(req) self.assertEqual(status.split()[0], '412') self.assertEqual(body, '') # sanity req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'HEAD'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPServerError, {}, None) status, headers, body = self.call_swift3(req) self.assertEqual(status.split()[0], '500') self.assertEqual(body, '') # sanity req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'HEAD'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPServiceUnavailable, {}, None) status, headers, body = self.call_swift3(req) self.assertEqual(status.split()[0], '500') self.assertEqual(body, '') # sanity
def test_caching(self): fail_to_resolve = ['CNAME lookup failed to resolve to a valid domain'] class memcache_stub(object): def __init__(self): self.cache = {} def get(self, key): return self.cache.get(key, None) def set(self, key, value, *a, **kw): self.cache[key] = value module = 'swift.common.middleware.cname_lookup.lookup_cname' dns_module = 'dns.resolver.Resolver.query' memcache = memcache_stub() with mock.patch(module) as m: m.return_value = (3600, 'c.example.com') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.cache': memcache}, headers={'Host': 'mysite2.com'}) resp = self.app(req.environ, start_response) self.assertEqual(resp, ['FAKE APP']) self.assertEqual(m.call_count, 1) self.assertEqual(memcache.cache.get('cname-mysite2.com'), 'c.example.com') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.cache': memcache}, headers={'Host': 'mysite2.com'}) resp = self.app(req.environ, start_response) self.assertEqual(resp, ['FAKE APP']) self.assertEqual(m.call_count, 1) self.assertEqual(memcache.cache.get('cname-mysite2.com'), 'c.example.com') for exc, num in ((dns.resolver.NXDOMAIN(), 3), (dns.resolver.NoAnswer(), 4)): with mock.patch(dns_module) as m: m.side_effect = exc req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.cache': memcache}, headers={'Host': 'mysite%d.com' % num}) resp = self.app(req.environ, start_response) self.assertEqual(resp, fail_to_resolve) self.assertEqual(m.call_count, 1) self.assertEqual(memcache.cache.get('cname-mysite3.com'), False) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.cache': memcache}, headers={'Host': 'mysite%d.com' % num}) resp = self.app(req.environ, start_response) self.assertEqual(resp, fail_to_resolve) self.assertEqual(m.call_count, 1) self.assertEqual( memcache.cache.get('cname-mysite%d.com' % num), False) with mock.patch(dns_module) as m: m.side_effect = dns.exception.DNSException() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.cache': memcache}, headers={'Host': 'mysite5.com'}) resp = self.app(req.environ, start_response) self.assertEqual(resp, fail_to_resolve) self.assertEqual(m.call_count, 1) self.assertFalse('cname-mysite5.com' in memcache.cache) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.cache': memcache}, headers={'Host': 'mysite5.com'}) resp = self.app(req.environ, start_response) self.assertEqual(resp, fail_to_resolve) self.assertEqual(m.call_count, 2) self.assertFalse('cname-mysite5.com' in memcache.cache)