def test_get_oversize_segment(self): # If we send a Content-Length header to the client, it's based on the # container listing. If a segment gets bigger by the time we get to it # (like if a client uploads a bigger segment w/the same name), we need # to not send anything beyond the length we promised. Also, we should # probably raise an exception. # This is now longer than the original seg_03+seg_04+seg_05 combined self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '20', 'Etag': 'seg03-etag'}, 'cccccccccccccccccccc') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, '200 OK') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check self.assertEqual(body, b'aaaaabbbbbccccccccccccccc') self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/mancon/manifest'), ('GET', '/v1/AUTH_test/c?prefix=seg'), ('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'), ('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'), ('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])
def _clean_outgoing_headers(self, headers): """ Removes any headers as per the middleware configuration for outgoing responses. :param headers: A WSGI start_response style list of headers, [('header1', 'value), ('header2', 'value), ...] :returns: The same headers list, but with some headers removed as per the middlware configuration for outgoing responses. """ headers = HeaderKeyDict(headers) for h in headers.keys(): if h in self.outgoing_allow_headers: continue for p in self.outgoing_allow_headers_startswith: if h.startswith(p): break else: if h in self.outgoing_remove_headers: del headers[h] continue for p in self.outgoing_remove_headers_startswith: if h.startswith(p): del headers[h] break return headers.items()
def test_get_undersize_segment(self): # If we send a Content-Length header to the client, it's based on the # container listing. If a segment gets smaller by the time we get to # it (like if a client uploads a smaller segment w/the same name), we # need to raise an exception so that the connection will be closed by # the WSGI server. Otherwise, the WSGI server will be waiting for the # next request, the client will still be waiting for the rest of the # response, and nobody will be happy. # Shrink it by a single byte self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")}, 'cccc') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) headers = HeaderKeyDict(headers) self.assertEqual(status, '200 OK') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee') self.assertTrue(isinstance(exc, exceptions.SegmentError))
def test_keys(self): headers = HeaderKeyDict() headers['content-length'] = 20 headers['cOnTent-tYpe'] = 'text/plain' headers['SomeThing-eLse'] = 'somevalue' self.assertEqual( set(headers.keys()), set(('Content-Length', 'Content-Type', 'Something-Else')))
def test_pop(self): headers = HeaderKeyDict() headers['content-length'] = 20 headers['cOntent-tYpe'] = 'text/plain' self.assertEqual(headers.pop('content-Length'), '20') self.assertEqual(headers.pop('Content-type'), 'text/plain') self.assertEqual(headers.pop('Something-Else', 'somevalue'), 'somevalue')
def process_object_update(self, update_path, device, policy): """ Process the object information to be updated and update. :param update_path: path to pickled object update file :param device: path to device :param policy: storage policy of object update """ try: update = pickle.load(open(update_path, 'rb')) except Exception: self.logger.exception( _('ERROR Pickle problem, quarantining %s'), update_path) self.stats.quarantines += 1 self.logger.increment('quarantines') target_path = os.path.join(device, 'quarantined', 'objects', os.path.basename(update_path)) renamer(update_path, target_path, fsync=False) return successes = update.get('successes', []) part, nodes = self.get_container_ring().get_nodes( update['account'], update['container']) obj = '/%s/%s/%s' % \ (update['account'], update['container'], update['obj']) headers_out = HeaderKeyDict(update['headers']) headers_out['user-agent'] = 'object-updater %s' % os.getpid() headers_out.setdefault('X-Backend-Storage-Policy-Index', str(int(policy))) events = [spawn(self.object_update, node, part, update['op'], obj, headers_out) for node in nodes if node['id'] not in successes] success = True new_successes = False for event in events: event_success, node_id = event.wait() if event_success is True: successes.append(node_id) new_successes = True else: success = False if success: self.stats.successes += 1 self.logger.increment('successes') self.logger.debug('Update sent for %(obj)s %(path)s', {'obj': obj, 'path': update_path}) self.stats.unlinks += 1 self.logger.increment('unlinks') os.unlink(update_path) else: self.stats.failures += 1 self.logger.increment('failures') self.logger.debug('Update failed for %(obj)s %(path)s', {'obj': obj, 'path': update_path}) if new_successes: update['successes'] = successes write_pickle(update, update_path, os.path.join( device, get_tmp_dir(policy)))
def test_head_large_object_too_many_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'HEAD'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) # etag is manifest's etag self.assertEqual(headers["Etag"], "etag-manyseg") self.assertIsNone(headers.get("Content-Length"))
def test_get_suffix_range_many_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=-5'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Range")) self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_range_many_segments_satisfiability_unknown(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=10-22'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") # this requires multiple pages of container listing, so we can't send # a Content-Length header self.assertIsNone(headers.get("Content-Length")) self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def handle_get_head_symlink(self, req): """ Handle get/head request when client sent parameter ?symlink=get :param req: HTTP GET or HEAD object request with param ?symlink=get :returns: Response Iterator """ resp = self._app_call(req.environ) response_header_dict = HeaderKeyDict(self._response_headers) symlink_sysmeta_to_usermeta(response_header_dict) self._response_headers = response_header_dict.items() return resp
def test_get_response_headers_with_legacy_data(self): broker = backend.AccountBroker(':memory:', account='a') now = time.time() with mock.patch('time.time', new=lambda: now): broker.initialize(Timestamp(now).internal) # add some container data ts = (Timestamp(t).internal for t in itertools.count(int(now))) total_containers = 0 total_objects = 0 total_bytes = 0 for policy in POLICIES: delete_timestamp = next(ts) put_timestamp = next(ts) object_count = int(policy) bytes_used = int(policy) * 10 broker.put_container('c-%s' % policy.name, put_timestamp, delete_timestamp, object_count, bytes_used, int(policy)) total_containers += 1 total_objects += object_count total_bytes += bytes_used expected = HeaderKeyDict({ 'X-Account-Container-Count': total_containers, 'X-Account-Object-Count': total_objects, 'X-Account-Bytes-Used': total_bytes, 'X-Timestamp': Timestamp(now).normal, 'X-PUT-Timestamp': Timestamp(now).normal, }) for policy in POLICIES: prefix = 'X-Account-Storage-Policy-%s-' % policy.name expected[prefix + 'Object-Count'] = int(policy) expected[prefix + 'Bytes-Used'] = int(policy) * 10 orig_policy_stats = broker.get_policy_stats def stub_policy_stats(*args, **kwargs): policy_stats = orig_policy_stats(*args, **kwargs) for stats in policy_stats.values(): # legacy db's won't return container_count del stats['container_count'] return policy_stats broker.get_policy_stats = stub_policy_stats resp_headers = utils.get_response_headers(broker) per_policy_container_headers = [ h for h in resp_headers if h.lower().startswith('x-account-storage-policy-') and h.lower().endswith('-container-count')] self.assertFalse(per_policy_container_headers) for key, value in resp_headers.items(): expected_value = expected.pop(key) self.assertEqual(expected_value, str(value), 'value for %r was %r not %r' % ( key, value, expected_value)) self.assertFalse(expected)
def test_get_range(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=8-17'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "10") self.assertEqual(body, b'bbcccccddd') expected_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) self.assertEqual(headers.get("Etag"), expected_etag)
def test_get_multi_range(self): # DLO doesn't support multi-range GETs. The way that you express that # in HTTP is to return a 200 response containing the whole entity. req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=5-9,15-19'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Range")) self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def handle(self, req, start_response): app_resp = self._app_call(req.environ) if is_success(self._get_status_int()): # only decrypt body of 2xx responses headers = HeaderKeyDict(self._response_headers) content_type = headers.get('content-type', '').split(';', 1)[0] if content_type == 'application/json': app_resp = self.process_json_resp(req, app_resp) start_response(self._response_status, self._response_headers, self._response_exc_info) return app_resp
def test_get_undersize_segment_range(self): # Shrink it by a single byte self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")}, 'cccc') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=0-14'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') # sanity check self.assertEqual(headers.get('Content-Length'), '15') # sanity check self.assertEqual(body, b'aaaaabbbbbcccc')
class FakeConn(object): def __init__(self, status, headers=None, body='', **kwargs): self.status = status try: self.reason = RESPONSE_REASONS[self.status][0] except Exception: self.reason = 'Fake' self.body = body self.resp_headers = HeaderKeyDict() if headers: self.resp_headers.update(headers) self.etag = None def _update_raw_call_args(self, *args, **kwargs): capture_attrs = ('host', 'port', 'method', 'path', 'req_headers', 'query_string') for attr, value in zip(capture_attrs, args[:len(capture_attrs)]): setattr(self, attr, value) return self def getresponse(self): if self.etag: self.resp_headers['etag'] = str(self.etag.hexdigest()) if isinstance(self.status, Exception): raise self.status return self def getheader(self, header, default=None): return self.resp_headers.get(header, default) def getheaders(self): return self.resp_headers.items() def read(self, amt=None): if isinstance(self.body, six.StringIO): return self.body.read(amt) elif amt is None: return self.body else: return Exception('Not a StringIO entry') def send(self, data): if not self.etag: self.etag = md5() self.etag.update(data)
def __init__(self, status, body='', headers=None, frag_index=None): self.status = status self.body = body self.readable = BytesIO(body) self.headers = HeaderKeyDict(headers) if frag_index is not None: self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index fake_reason = ('Fake', 'This response is a lie.') self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0]
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req_headers = swob.Request(env).headers self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ( (method, path),)) self._calls.append((method, path, req_headers)) # simulate object PUT if method == 'PUT' and obj: input = env['wsgi.input'].read() etag = md5(input).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(input)) # keep it for subsequent GET requests later self.uploaded[path] = (deepcopy(headers), input) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # range requests ought to work, which require conditional_response=True req = swob.Request(env) resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD')) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def __init__(self, status, headers=None, body='', **kwargs): self.status = status try: self.reason = RESPONSE_REASONS[self.status][0] except Exception: self.reason = 'Fake' self.body = body self.resp_headers = HeaderKeyDict() if headers: self.resp_headers.update(headers) self.etag = None
def __call__(self, env, start_response): method = env["REQUEST_METHOD"] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env["PATH_INFO"] _, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True) if env.get("QUERY_STRING"): path += "?" + env["QUERY_STRING"] if "swift.authorize" in env: resp = env["swift.authorize"](swob.Request(env)) if resp: return resp(env, start_response) req_headers = swob.Request(env).headers self.swift_sources.append(env.get("swift.source")) self.txn_ids.append(env.get("swift.trans_id")) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses: resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"]) headers = HeaderKeyDict(raw_headers) elif method == "HEAD" and ("GET", path) in self._responses: resp_class, raw_headers, body = self._find_response("GET", path) body = None headers = HeaderKeyDict(raw_headers) elif method == "GET" and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path),)) self._calls.append((method, path, req_headers)) # simulate object PUT if method == "PUT" and obj: input = env["wsgi.input"].read() etag = md5(input).hexdigest() headers.setdefault("Etag", etag) headers.setdefault("Content-Length", len(input)) # keep it for subsequent GET requests later self.uploaded[path] = (deepcopy(headers), input) if "CONTENT_TYPE" in env: self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"] # range requests ought to work, which require conditional_response=True req = swob.Request(env) resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD")) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def getheaders(self): etag = self.etag if not etag: if isinstance(self.body, str): etag = '"' + md5(self.body).hexdigest() + '"' else: etag = '"68b329da9893e34099c7d8ad5cb9c940"' headers = HeaderKeyDict({ 'content-length': len(self.body), 'content-type': 'x-application/test', 'x-timestamp': self.timestamp, 'x-backend-timestamp': self.timestamp, 'last-modified': self.timestamp, 'x-object-meta-test': 'testing', 'x-delete-at': '9876543210', 'etag': etag, 'x-works': 'yes', }) if self.status // 100 == 2: headers['x-account-container-count'] = \ kwargs.get('count', 12345) if not self.timestamp: # when timestamp is None, HeaderKeyDict raises KeyError headers.pop('x-timestamp', None) try: if next(container_ts_iter) is False: headers['x-container-timestamp'] = '1' except StopIteration: pass am_slow, value = self.get_slow() if am_slow: headers['content-length'] = '4' headers.update(self.headers) return headers.items()
class StubResponse(object): def __init__(self, status, body='', headers=None, frag_index=None): self.status = status self.body = body self.readable = BytesIO(body) self.headers = HeaderKeyDict(headers) if frag_index is not None: self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index fake_reason = ('Fake', 'This response is a lie.') self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0] def getheader(self, header_name, default=None): return self.headers.get(header_name, default) def getheaders(self): if 'Content-Length' not in self.headers: self.headers['Content-Length'] = len(self.body) return self.headers.items() def read(self, amt=0): return self.readable.read(amt)
def test_direct_head_container(self): headers = HeaderKeyDict(key='value') with mocked_http_conn(200, headers) as conn: resp = direct_client.direct_head_container(self.node, self.part, self.account, self.container) self.assertEqual(conn.host, self.node['ip']) self.assertEqual(conn.port, self.node['port']) self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(headers, resp)
def test_obj_put_legacy_updates(self): ts = (normalize_timestamp(t) for t in itertools.count(int(time()))) policy = POLICIES.get_by_index(0) # setup updater conf = { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, } async_dir = os.path.join(self.sda1, get_async_dir(policy)) os.mkdir(async_dir) account, container, obj = 'a', 'c', 'o' # write an async for op in ('PUT', 'DELETE'): self.logger._clear() daemon = object_updater.ObjectUpdater(conf, logger=self.logger) dfmanager = DiskFileManager(conf, daemon.logger) # don't include storage-policy-index in headers_out pickle headers_out = HeaderKeyDict({ 'x-size': 0, 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-timestamp': next(ts), }) data = {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out} dfmanager.pickle_async_update(self.sda1, account, container, obj, data, next(ts), policy) request_log = [] def capture(*args, **kwargs): request_log.append((args, kwargs)) # run once fake_status_codes = [200, 200, 200] with mocked_http_conn(*fake_status_codes, give_connect=capture): daemon.run_once() self.assertEqual(len(fake_status_codes), len(request_log)) for request_args, request_kwargs in request_log: ip, part, method, path, headers, qs, ssl = request_args self.assertEqual(method, op) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], str(int(policy))) self.assertEqual(daemon.logger.get_increment_counts(), {'successes': 1, 'unlinks': 1, 'async_pendings': 1})
def test_head_large_object_no_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-no-segments', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(headers["Etag"], '"%s"' % md5hex("")) self.assertEqual(headers["Content-Length"], "0") # one request to HEAD the manifest # one request for the first page of listings # *zero* requests for the second page of listings self.assertEqual( self.app.calls, [('HEAD', '/v1/AUTH_test/mancon/manifest-no-segments'), ('GET', '/v1/AUTH_test/c?prefix=noseg_')])
def test_get_undersize_segment(self): # If we send a Content-Length header to the client, it's based on the # container listing. If a segment gets smaller by the time we get to # it (like if a client uploads a smaller segment w/the same name), we # need to raise an exception so that the connection will be closed by # the WSGI server. Otherwise, the WSGI server will be waiting for the # next request, the client will still be waiting for the rest of the # response, and nobody will be happy. # Shrink it by a single byte self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")}, 'cccc') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, '200 OK') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check self.assertEqual(body, b'aaaaabbbbbccccdddddeeeee')
def test_direct_put_object_header_content_length(self): contents = six.StringIO('123456') stub_headers = HeaderKeyDict({'Content-Length': '6'}) with mocked_http_conn(200) as conn: resp = direct_client.direct_put_object(self.node, self.part, self.account, self.container, self.obj, contents, headers=stub_headers) self.assertEqual('PUT', conn.method) self.assertEqual(conn.req_headers['Content-length'], '6') self.assertEqual(md5('123456').hexdigest(), resp)
def __init__(self, stype, method, node, part, path, resp, host=None): # host can be used to override the node ip and port reported in # the exception host = host if host is not None else node full_path = quote('/%s/%s%s' % (node['device'], part, path)) msg = '%s server %s:%s direct %s %r gave status %s' % ( stype, host['ip'], host['port'], method, full_path, resp.status) headers = HeaderKeyDict(resp.getheaders()) super(DirectClientException, self).__init__(msg, http_host=host['ip'], http_port=host['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason, http_headers=headers)
def test_direct_head_object(self): headers = HeaderKeyDict({'x-foo': 'bar'}) with mocked_http_conn(200, headers) as conn: resp = direct_client.direct_head_object( self.node, self.part, self.account, self.container, self.obj, headers=headers) self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.obj_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual('bar', conn.req_headers.get('x-foo')) self.assertTrue('x-timestamp' not in conn.req_headers, 'x-timestamp was in HEAD request headers') self.assertEqual(headers, resp)
def test_get_manifest_passthrough(self): # reregister it with the query param self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest?multipart-manifest=get', swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag', 'X-Object-Manifest': 'c/seg'}, 'manifest-contents') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING': 'multipart-manifest=get'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(headers["Etag"], "manifest-etag") self.assertEqual(body, "manifest-contents")
def test_error_fetching_second_segment(self): self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPForbidden, {}, None) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertEqual(''.join(body), "aaaaa") # first segment made it out self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [ 'While processing manifest /v1/AUTH_test/mancon/manifest, ' 'got 403 while retrieving /v1/AUTH_test/c/seg_02', ])
def test_error_fetching_second_segment(self): self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPForbidden, {}, None) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) headers = HeaderKeyDict(headers) self.assertTrue(isinstance(exc, exceptions.SegmentError)) self.assertEqual(status, "200 OK") self.assertEqual(''.join(body), "aaaaa") # first segment made it out err_lines = self.dlo.logger.get_lines_for_level('error') self.assertEqual(len(err_lines), 1) self.assertTrue(err_lines[0].startswith( 'ERROR: An error occurred while retrieving segments'))
def test_account_listing_response(self): req = Request.blank('') now = time.time() with mock.patch('time.time', new=lambda: now): resp = utils.account_listing_response('a', req, 'text/plain') self.assertEqual(resp.status_int, 204) expected = HeaderKeyDict({ 'Content-Type': 'text/plain; charset=utf-8', 'X-Account-Container-Count': 0, 'X-Account-Object-Count': 0, 'X-Account-Bytes-Used': 0, 'X-Timestamp': Timestamp(now).normal, 'X-PUT-Timestamp': Timestamp(now).normal, }) self.assertEqual(expected, resp.headers) self.assertEqual(b'', resp.body)
def direct_get_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15, resp_chunk_size=None, headers=None): """ Get object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param resp_chunk_size: if defined, chunk size of data to read. :param headers: dict to be passed into HTTPConnection headers :returns: a tuple of (response headers, the object's contents) The response headers will be a HeaderKeyDict. :raises ClientException: HTTP GET request failed """ if headers is None: headers = {} ip, port = get_ip_port(node, headers) path = _make_path(account, container, obj) with Timeout(conn_timeout): conn = http_connect(ip, port, node['device'], part, 'GET', path, headers=gen_headers(headers)) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): resp.read() raise DirectClientException('Object', 'GET', node, part, path, resp) if resp_chunk_size: def _object_body(): buf = resp.read(resp_chunk_size) while buf: yield buf buf = resp.read(resp_chunk_size) object_body = _object_body() else: object_body = resp.read() resp_headers = HeaderKeyDict() for header, value in resp.getheaders(): resp_headers[header] = value return resp_headers, object_body
def _get_direct_account_container(path, stype, node, part, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15, end_marker=None, reverse=None, headers=None): """Base class for get direct account and container. Do not use directly use the get_direct_account or get_direct_container instead. """ if headers is None: headers = {} params = ['format=json'] if marker: params.append('marker=%s' % quote(marker)) if limit: params.append('limit=%d' % limit) if prefix: params.append('prefix=%s' % quote(prefix)) if delimiter: params.append('delimiter=%s' % quote(delimiter)) if end_marker: params.append('end_marker=%s' % quote(end_marker)) if reverse: params.append('reverse=%s' % quote(reverse)) qs = '&'.join(params) ip, port = get_ip_port(node, headers) with Timeout(conn_timeout): conn = http_connect(ip, port, node['device'], part, 'GET', path, query_string=qs, headers=gen_headers(hdrs_in=headers)) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): resp.read() raise DirectClientException(stype, 'GET', node, part, path, resp) resp_headers = HeaderKeyDict() for header, value in resp.getheaders(): resp_headers[header] = value if resp.status == HTTP_NO_CONTENT: resp.read() return resp_headers, [] return resp_headers, json.loads(resp.read())
def _get_direct_account_container(path, stype, node, part, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15): """Base class for get direct account and container. Do not use directly use the get_direct_account or get_direct_container instead. """ qs = 'format=json' if marker: qs += '&marker=%s' % quote(marker) if limit: qs += '&limit=%d' % limit if prefix: qs += '&prefix=%s' % quote(prefix) if delimiter: qs += '&delimiter=%s' % quote(delimiter) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, query_string=qs, headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): resp.read() raise DirectClientException(stype, 'GET', node, part, path, resp) resp_headers = HeaderKeyDict() for header, value in resp.getheaders(): resp_headers[header] = value if resp.status == HTTP_NO_CONTENT: resp.read() return resp_headers, [] return resp_headers, json.loads(resp.read())
def test_set(self): # mappings = ((<tuple of input vals>, <expected output val>), ...) mappings = (((1.618, '1.618', b'1.618', u'1.618'), '1.618'), ((20, '20', b'20', u'20'), '20'), ((True, 'True', b'True', u'True'), 'True'), ((False, 'False', b'False', u'False'), 'False')) for vals, expected in mappings: for val in vals: headers = HeaderKeyDict(test=val) actual = headers['test'] self.assertEqual( expected, actual, 'Expected %s but got %s for val %s' % (expected, actual, val)) self.assertIsInstance( actual, str, 'Expected type str but got %s for val %s of type %s' % (type(actual), val, type(val)))
def test_setdefault(self): headers = HeaderKeyDict() # it gets set headers.setdefault("x-rubber-ducky", "the one") self.assertEqual(headers["X-Rubber-Ducky"], "the one") # it has the right return value ret = headers.setdefault("x-boat", "dinghy") self.assertEqual(ret, "dinghy") ret = headers.setdefault("x-boat", "yacht") self.assertEqual(ret, "dinghy") # shouldn't crash headers.setdefault("x-sir-not-appearing-in-this-request", None)
def test_setdefault(self): headers = HeaderKeyDict() # it gets set headers.setdefault('x-rubber-ducky', 'the one') self.assertEqual(headers['X-Rubber-Ducky'], 'the one') # it has the right return value ret = headers.setdefault('x-boat', 'dinghy') self.assertEqual(ret, 'dinghy') ret = headers.setdefault('x-boat', 'yacht') self.assertEqual(ret, 'dinghy') # shouldn't crash headers.setdefault('x-sir-not-appearing-in-this-request', None)
def test_direct_put_object_header_content_length(self): contents = io.BytesIO(b'123456') stub_headers = HeaderKeyDict({'Content-Length': '6'}) with mocked_http_conn(200) as conn: resp = direct_client.direct_put_object(self.node, self.part, self.account, self.container, self.obj, contents, headers=stub_headers) self.assertEqual(conn.host, self.node['ip']) self.assertEqual(conn.port, self.node['port']) self.assertEqual('PUT', conn.method) self.assertEqual(conn.req_headers['Content-length'], '6') self.assertEqual( md5(b'123456', usedforsecurity=False).hexdigest(), resp)
def test_direct_head_container_deleted(self): important_timestamp = Timestamp.now().internal headers = HeaderKeyDict( {'X-Backend-Important-Timestamp': important_timestamp}) with mocked_http_conn(404, headers) as conn: with self.assertRaises(ClientException) as raised: direct_client.direct_head_container(self.node, self.part, self.account, self.container) self.assertEqual(conn.host, self.node['ip']) self.assertEqual(conn.port, self.node['port']) self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(raised.exception.http_status, 404) self.assertEqual(raised.exception.http_headers, headers)
def test_direct_head_container_error(self): headers = HeaderKeyDict(key='value') with mocked_http_conn(503, headers) as conn: with self.assertRaises(ClientException) as raised: direct_client.direct_head_container(self.node, self.part, self.account, self.container) # check request self.assertEqual(conn.host, self.node['ip']) self.assertEqual(conn.port, self.node['port']) self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(raised.exception.http_status, 503) self.assertEqual(raised.exception.http_headers, headers) self.assertTrue('HEAD' in str(raised.exception))
def test_error_fetching_second_segment(self): self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPForbidden, {}, None) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") # first segment made it out self.assertEqual(body, b'aaaaa') self.assertEqual(self.app.unread_requests, {}) self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [ 'While processing manifest /v1/AUTH_test/mancon/manifest, ' 'got 403 (<html><h1>Forbidden</h1><p>Access was denied to this ' 'reso...) while retrieving /v1/AUTH_test/c/seg_02', ])
def test_direct_head_container_deleted(self): important_timestamp = Timestamp(time.time()).internal headers = HeaderKeyDict({'X-Backend-Important-Timestamp': important_timestamp}) with mocked_http_conn(404, headers) as conn: try: direct_client.direct_head_container( self.node, self.part, self.account, self.container) except Exception as err: self.assertTrue(isinstance(err, ClientException)) else: self.fail('ClientException not raised') self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(err.http_status, 404) self.assertEqual(err.http_headers, headers)
def test_direct_head_container_replication_net(self): headers = HeaderKeyDict(key='value') with mocked_http_conn(200, headers) as conn: resp = direct_client.direct_head_container( self.node, self.part, self.account, self.container, headers={'X-Backend-Use-Replication-Network': 'on'}) self.assertEqual(conn.host, self.node['replication_ip']) self.assertEqual(conn.port, self.node['replication_port']) self.assertNotEqual(conn.host, self.node['ip']) self.assertNotEqual(conn.port, self.node['port']) self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(headers, resp)
def test_mismatched_etag_fetching_second_segment(self): self.app.register( 'GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")}, 'WRONG') req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertEqual(headers['Content-Length'], "25") # stop after error self.assertEqual(body, b"aaaaaWRONG") log_lines = self.dlo.logger.get_lines_for_level('error') self.assertEqual(len(log_lines), 1, 'Expected one log line, got %r' % log_lines) self.assertEqual(log_lines[0][:21], 'Bad MD5 checksum for ')
def test_direct_head_container_error(self): headers = HeaderKeyDict(key='value') with mocked_http_conn(503, headers) as conn: try: direct_client.direct_head_container( self.node, self.part, self.account, self.container) except ClientException as err: pass else: self.fail('ClientException not raised') # check request self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(err.http_status, 503) self.assertEqual(err.http_headers, headers) self.assertTrue('HEAD' in str(err))
def test_direct_get_container(self): headers = HeaderKeyDict({'key': 'value'}) body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]' with mocked_http_conn(200, headers, body) as conn: resp_headers, resp = direct_client.direct_get_container( self.node, self.part, self.account, self.container, marker='marker', prefix='prefix', delimiter='delimiter', limit=1000) self.assertEqual(conn.req_headers['user-agent'], 'direct-client %s' % os.getpid()) self.assertEqual(headers, resp_headers) self.assertEqual(json.loads(body), resp) self.assertTrue('marker=marker' in conn.query_string) self.assertTrue('delimiter=delimiter' in conn.query_string) self.assertTrue('limit=1000' in conn.query_string) self.assertTrue('prefix=prefix' in conn.query_string) self.assertTrue('format=json' in conn.query_string)
def accepter(sock, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write(b'HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEqual(inc.readline(), b'PUT /sda1/0/a/c/o HTTP/1.1\r\n') headers = HeaderKeyDict() line = bytes_to_wsgi(inc.readline()) while line and line != '\r\n': headers[line.split(':')[0]] = \ line.split(':')[1].strip() line = bytes_to_wsgi(inc.readline()) self.assertIn('x-container-timestamp', headers) self.assertIn('X-Backend-Storage-Policy-Index', headers) except BaseException as err: return err return None
def test_mismatched_length_fetching_second_segment(self): self.app.register( 'GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbb")}, # Use a list so we can get a discrepency between content-length and # number of bytes in the app_iter [b'b' * 4]) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertEqual(headers['Content-Length'], "25") # stop after error self.assertEqual(body, b"aaaaabbbb") log_lines = self.dlo.logger.get_lines_for_level('error') self.assertEqual(len(log_lines), 1, 'Expected one log line, got %r' % log_lines) self.assertEqual(log_lines[0][:24], 'Bad response length for ')
def test_etag_comparison_ignores_quotes(self): # a little future-proofing here in case we ever fix this in swob self.app.register( 'HEAD', '/v1/AUTH_test/mani/festo', swob.HTTPOk, {'Content-Length': '0', 'Etag': 'blah', 'X-Object-Manifest': 'c/quotetags'}, None) self.app.register( 'GET', '/v1/AUTH_test/c?prefix=quotetags', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps([{"hash": "\"abc\"", "bytes": 5, "name": "quotetags1", "last_modified": "2013-11-22T02:42:14.261620", "content-type": "application/octet-stream"}, {"hash": "def", "bytes": 5, "name": "quotetags2", "last_modified": "2013-11-22T02:42:14.261620", "content-type": "application/octet-stream"}])) req = swob.Request.blank('/v1/AUTH_test/mani/festo', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(headers["Etag"], '"' + hashlib.md5("abcdef").hexdigest() + '"')
def direct_head_container(node, part, account, container, conn_timeout=5, response_timeout=15): """ Request container information directly from the container server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: a dict containing the response's headers in a HeaderKeyDict :raises ClientException: HTTP HEAD request failed """ path = '/%s/%s' % (account, container) resp = _make_req(node, part, 'HEAD', path, gen_headers(), 'Container', conn_timeout, response_timeout) resp_headers = HeaderKeyDict() for header, value in resp.getheaders(): resp_headers[header] = value return resp_headers
def resolve_etag_is_at_header(req, metadata): """ Helper function to resolve an alternative etag value that may be stored in metadata under an alternate name. The value of the request's X-Backend-Etag-Is-At header (if it exists) is a comma separated list of alternate names in the metadata at which an alternate etag value may be found. This list is processed in order until an alternate etag is found. The left most value in X-Backend-Etag-Is-At will have been set by the left most middleware, or if no middleware, by ECObjectController, if an EC policy is in use. The left most middleware is assumed to be the authority on what the etag value of the object content is. The resolver will work from left to right in the list until it finds a value that is a name in the given metadata. So the left most wins, IF it exists in the metadata. By way of example, assume the encrypter middleware is installed. If an object is *not* encrypted then the resolver will not find the encrypter middleware's alternate etag sysmeta (X-Object-Sysmeta-Crypto-Etag) but will then find the EC alternate etag (if EC policy). But if the object *is* encrypted then X-Object-Sysmeta-Crypto-Etag is found and used, which is correct because it should be preferred over X-Object-Sysmeta-Ec-Etag. :param req: a swob Request :param metadata: a dict containing object metadata :return: an alternate etag value if any is found, otherwise None """ alternate_etag = None metadata = HeaderKeyDict(metadata) if "X-Backend-Etag-Is-At" in req.headers: names = list_from_csv(req.headers["X-Backend-Etag-Is-At"]) for name in names: if name in metadata: alternate_etag = metadata[name] break return alternate_etag
def test_get_manifest(self): expected_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(headers["Etag"], expected_etag) self.assertEqual(headers["Content-Length"], "25") self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee') for _, _, hdrs in self.app.calls_with_headers[1:]: ua = hdrs.get("User-Agent", "") self.assertTrue("DLO MultipartGET" in ua) self.assertFalse("DLO MultipartGET DLO MultipartGET" in ua) # the first request goes through unaltered self.assertFalse( "DLO MultipartGET" in self.app.calls_with_headers[0][2]) # we set swift.source for everything but the first request self.assertEqual(self.app.swift_sources, [None, 'DLO', 'DLO', 'DLO', 'DLO', 'DLO', 'DLO'])
def getheaders(self): etag = self.etag if not etag: if isinstance(self.body, str): etag = '"' + md5(self.body).hexdigest() + '"' else: etag = '"68b329da9893e34099c7d8ad5cb9c940"' headers = HeaderKeyDict( { "content-length": len(self.body), "content-type": "x-application/test", "x-timestamp": self.timestamp, "x-backend-timestamp": self.timestamp, "last-modified": self.timestamp, "x-object-meta-test": "testing", "x-delete-at": "9876543210", "etag": etag, "x-works": "yes", } ) if self.status // 100 == 2: headers["x-account-container-count"] = kwargs.get("count", 12345) if not self.timestamp: # when timestamp is None, HeaderKeyDict raises KeyError headers.pop("x-timestamp", None) try: if next(container_ts_iter) is False: headers["x-container-timestamp"] = "1" except StopIteration: pass am_slow, value = self.get_slow() if am_slow: headers["content-length"] = "4" headers.update(self.headers) return headers.items()
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ( (method, path),)) # simulate object PUT if method == 'PUT' and obj: put_body = ''.join(iter(env['wsgi.input'].read, '')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if (is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) backend_etag_header = req.headers.get('X-Backend-Etag-Is-At') conditional_etag = None if backend_etag_header and backend_etag_header in headers: # Apply conditional etag overrides conditional_etag = headers[backend_etag_header] # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class( req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def __call__(self, env, start_response): method = env["REQUEST_METHOD"] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env["PATH_INFO"] _, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True) if env.get("QUERY_STRING"): path += "?" + env["QUERY_STRING"] if "swift.authorize" in env: resp = env["swift.authorize"](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get("swift.source")) self.txn_ids.append(env.get("swift.trans_id")) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses: resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"]) headers = HeaderKeyDict(raw_headers) elif method == "HEAD" and ("GET", path) in self._responses: resp_class, raw_headers, body = self._find_response("GET", path) body = None headers = HeaderKeyDict(raw_headers) elif method == "GET" and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path),)) # simulate object PUT if method == "PUT" and obj: put_body = "".join(iter(env["wsgi.input"].read, "")) if "swift.callback.update_footers" in env: footers = HeaderKeyDict() env["swift.callback.update_footers"](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault("Etag", etag) headers.setdefault("Content-Length", len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"] # simulate object POST elif method == "POST" and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta("object", k) and not is_object_transient_sysmeta(k)) ) # apply from new new_metadata.update( dict( (k, v) for k, v in req.headers.items() if (is_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower == "content-type") ) ) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append((method, path, HeaderKeyDict(req.headers))) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ("GET", "HEAD") ) else: resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD")) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def test_keys(self): headers = HeaderKeyDict() headers["content-length"] = 20 headers["cOnTent-tYpe"] = "text/plain" headers["SomeThing-eLse"] = "somevalue" self.assertEqual(set(headers.keys()), set(("Content-Length", "Content-Type", "Something-Else")))
def test_update(self): headers = HeaderKeyDict() headers.update({'Content-Length': '0'}) headers.update([('Content-Type', 'text/plain')]) self.assertEqual(headers['Content-Length'], '0') self.assertEqual(headers['Content-Type'], 'text/plain')
def test_get(self): headers = HeaderKeyDict() headers['content-length'] = 20 self.assertEqual(headers.get('CONTENT-LENGTH'), '20') self.assertEqual(headers.get('something-else'), None) self.assertEqual(headers.get('something-else', True), True)