def test_get_oversize_segment(self):
# If we send a Content-Length header to the client, it's based on the
# container listing. If a segment gets bigger by the time we get to it
# (like if a client uploads a bigger segment w/the same name), we need
# to not send anything beyond the length we promised. Also, we should
# probably raise an exception.
# This is now longer than the original seg_03+seg_04+seg_05 combined
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '20', 'Etag': 'seg03-etag'},
'cccccccccccccccccccc')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
self.assertEqual(
self.app.calls,
[('GET', '/v1/AUTH_test/mancon/manifest'),
('GET', '/v1/AUTH_test/c?prefix=seg'),
('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])
def _clean_outgoing_headers(self, headers):
"""
Removes any headers as per the middleware configuration for
outgoing responses.
:param headers: A WSGI start_response style list of headers,
[('header1', 'value), ('header2', 'value),
...]
:returns: The same headers list, but with some headers
removed as per the middlware configuration for
outgoing responses.
"""
headers = HeaderKeyDict(headers)
for h in headers.keys():
if h in self.outgoing_allow_headers:
continue
for p in self.outgoing_allow_headers_startswith:
if h.startswith(p):
break
else:
if h in self.outgoing_remove_headers:
del headers[h]
continue
for p in self.outgoing_remove_headers_startswith:
if h.startswith(p):
del headers[h]
break
return headers.items()
def test_get_undersize_segment(self):
# If we send a Content-Length header to the client, it's based on the
# container listing. If a segment gets smaller by the time we get to
# it (like if a client uploads a smaller segment w/the same name), we
# need to raise an exception so that the connection will be closed by
# the WSGI server. Otherwise, the WSGI server will be waiting for the
# next request, the client will still be waiting for the rest of the
# response, and nobody will be happy.
# Shrink it by a single byte
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
'cccc')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body, exc = self.call_dlo(req, expect_exception=True)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee')
self.assertTrue(isinstance(exc, exceptions.SegmentError))
def test_keys(self):
headers = HeaderKeyDict()
headers['content-length'] = 20
headers['cOnTent-tYpe'] = 'text/plain'
headers['SomeThing-eLse'] = 'somevalue'
self.assertEqual(
set(headers.keys()),
set(('Content-Length', 'Content-Type', 'Something-Else')))
def test_pop(self):
headers = HeaderKeyDict()
headers['content-length'] = 20
headers['cOntent-tYpe'] = 'text/plain'
self.assertEqual(headers.pop('content-Length'), '20')
self.assertEqual(headers.pop('Content-type'), 'text/plain')
self.assertEqual(headers.pop('Something-Else', 'somevalue'),
'somevalue')
def process_object_update(self, update_path, device, policy):
"""
Process the object information to be updated and update.
:param update_path: path to pickled object update file
:param device: path to device
:param policy: storage policy of object update
"""
try:
update = pickle.load(open(update_path, 'rb'))
except Exception:
self.logger.exception(
_('ERROR Pickle problem, quarantining %s'), update_path)
self.stats.quarantines += 1
self.logger.increment('quarantines')
target_path = os.path.join(device, 'quarantined', 'objects',
os.path.basename(update_path))
renamer(update_path, target_path, fsync=False)
return
successes = update.get('successes', [])
part, nodes = self.get_container_ring().get_nodes(
update['account'], update['container'])
obj = '/%s/%s/%s' % \
(update['account'], update['container'], update['obj'])
headers_out = HeaderKeyDict(update['headers'])
headers_out['user-agent'] = 'object-updater %s' % os.getpid()
headers_out.setdefault('X-Backend-Storage-Policy-Index',
str(int(policy)))
events = [spawn(self.object_update,
node, part, update['op'], obj, headers_out)
for node in nodes if node['id'] not in successes]
success = True
new_successes = False
for event in events:
event_success, node_id = event.wait()
if event_success is True:
successes.append(node_id)
new_successes = True
else:
success = False
if success:
self.stats.successes += 1
self.logger.increment('successes')
self.logger.debug('Update sent for %(obj)s %(path)s',
{'obj': obj, 'path': update_path})
self.stats.unlinks += 1
self.logger.increment('unlinks')
os.unlink(update_path)
else:
self.stats.failures += 1
self.logger.increment('failures')
self.logger.debug('Update failed for %(obj)s %(path)s',
{'obj': obj, 'path': update_path})
if new_successes:
update['successes'] = successes
write_pickle(update, update_path, os.path.join(
device, get_tmp_dir(policy)))
def test_head_large_object_too_many_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'HEAD'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
# etag is manifest's etag
self.assertEqual(headers["Etag"], "etag-manyseg")
self.assertIsNone(headers.get("Content-Length"))
def test_get_suffix_range_many_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=-5'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_range_many_segments_satisfiability_unknown(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=10-22'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
# this requires multiple pages of container listing, so we can't send
# a Content-Length header
self.assertIsNone(headers.get("Content-Length"))
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def handle_get_head_symlink(self, req):
"""
Handle get/head request when client sent parameter ?symlink=get
:param req: HTTP GET or HEAD object request with param ?symlink=get
:returns: Response Iterator
"""
resp = self._app_call(req.environ)
response_header_dict = HeaderKeyDict(self._response_headers)
symlink_sysmeta_to_usermeta(response_header_dict)
self._response_headers = response_header_dict.items()
return resp
def test_get_response_headers_with_legacy_data(self):
broker = backend.AccountBroker(':memory:', account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
# add some container data
ts = (Timestamp(t).internal for t in itertools.count(int(now)))
total_containers = 0
total_objects = 0
total_bytes = 0
for policy in POLICIES:
delete_timestamp = next(ts)
put_timestamp = next(ts)
object_count = int(policy)
bytes_used = int(policy) * 10
broker.put_container('c-%s' % policy.name, put_timestamp,
delete_timestamp, object_count, bytes_used,
int(policy))
total_containers += 1
total_objects += object_count
total_bytes += bytes_used
expected = HeaderKeyDict({
'X-Account-Container-Count': total_containers,
'X-Account-Object-Count': total_objects,
'X-Account-Bytes-Used': total_bytes,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
})
for policy in POLICIES:
prefix = 'X-Account-Storage-Policy-%s-' % policy.name
expected[prefix + 'Object-Count'] = int(policy)
expected[prefix + 'Bytes-Used'] = int(policy) * 10
orig_policy_stats = broker.get_policy_stats
def stub_policy_stats(*args, **kwargs):
policy_stats = orig_policy_stats(*args, **kwargs)
for stats in policy_stats.values():
# legacy db's won't return container_count
del stats['container_count']
return policy_stats
broker.get_policy_stats = stub_policy_stats
resp_headers = utils.get_response_headers(broker)
per_policy_container_headers = [
h for h in resp_headers if
h.lower().startswith('x-account-storage-policy-') and
h.lower().endswith('-container-count')]
self.assertFalse(per_policy_container_headers)
for key, value in resp_headers.items():
expected_value = expected.pop(key)
self.assertEqual(expected_value, str(value),
'value for %r was %r not %r' % (
key, value, expected_value))
self.assertFalse(expected)
def test_get_range(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=8-17'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "10")
self.assertEqual(body, b'bbcccccddd')
expected_etag = '"%s"' % md5hex(
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
md5hex("ddddd") + md5hex("eeeee"))
self.assertEqual(headers.get("Etag"), expected_etag)
def test_get_multi_range(self):
# DLO doesn't support multi-range GETs. The way that you express that
# in HTTP is to return a 200 response containing the whole entity.
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=5-9,15-19'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def handle(self, req, start_response):
app_resp = self._app_call(req.environ)
if is_success(self._get_status_int()):
# only decrypt body of 2xx responses
headers = HeaderKeyDict(self._response_headers)
content_type = headers.get('content-type', '').split(';', 1)[0]
if content_type == 'application/json':
app_resp = self.process_json_resp(req, app_resp)
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return app_resp
def test_get_undersize_segment_range(self):
# Shrink it by a single byte
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
'cccc')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=0-14'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '206 Partial Content') # sanity check
self.assertEqual(headers.get('Content-Length'), '15') # sanity check
self.assertEqual(body, b'aaaaabbbbbcccc')
class FakeConn(object):
def __init__(self, status, headers=None, body='', **kwargs):
self.status = status
try:
self.reason = RESPONSE_REASONS[self.status][0]
except Exception:
self.reason = 'Fake'
self.body = body
self.resp_headers = HeaderKeyDict()
if headers:
self.resp_headers.update(headers)
self.etag = None
def _update_raw_call_args(self, *args, **kwargs):
capture_attrs = ('host', 'port', 'method', 'path', 'req_headers',
'query_string')
for attr, value in zip(capture_attrs, args[:len(capture_attrs)]):
setattr(self, attr, value)
return self
def getresponse(self):
if self.etag:
self.resp_headers['etag'] = str(self.etag.hexdigest())
if isinstance(self.status, Exception):
raise self.status
return self
def getheader(self, header, default=None):
return self.resp_headers.get(header, default)
def getheaders(self):
return self.resp_headers.items()
def read(self, amt=None):
if isinstance(self.body, six.StringIO):
return self.body.read(amt)
elif amt is None:
return self.body
else:
return Exception('Not a StringIO entry')
def send(self, data):
if not self.etag:
self.etag = md5()
self.etag.update(data)
def __init__(self, status, body='', headers=None, frag_index=None):
self.status = status
self.body = body
self.readable = BytesIO(body)
self.headers = HeaderKeyDict(headers)
if frag_index is not None:
self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index
fake_reason = ('Fake', 'This response is a lie.')
self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0]
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
path = env['PATH_INFO']
_, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
rest_with_last=True)
if env.get('QUERY_STRING'):
path += '?' + env['QUERY_STRING']
if 'swift.authorize' in env:
resp = env['swift.authorize'](swob.Request(env))
if resp:
return resp(env, start_response)
req_headers = swob.Request(env).headers
self.swift_sources.append(env.get('swift.source'))
self.txn_ids.append(env.get('swift.trans_id'))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if (env.get('QUERY_STRING')
and (method, env['PATH_INFO']) in self._responses):
resp_class, raw_headers, body = self._find_response(
method, env['PATH_INFO'])
headers = HeaderKeyDict(raw_headers)
elif method == 'HEAD' and ('GET', path) in self._responses:
resp_class, raw_headers, body = self._find_response(
'GET', path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == 'GET' and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % (
(method, path),))
self._calls.append((method, path, req_headers))
# simulate object PUT
if method == 'PUT' and obj:
input = env['wsgi.input'].read()
etag = md5(input).hexdigest()
headers.setdefault('Etag', etag)
headers.setdefault('Content-Length', len(input))
# keep it for subsequent GET requests later
self.uploaded[path] = (deepcopy(headers), input)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
# range requests ought to work, which require conditional_response=True
req = swob.Request(env)
resp = resp_class(req=req, headers=headers, body=body,
conditional_response=req.method in ('GET', 'HEAD'))
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def __init__(self, status, headers=None, body='', **kwargs):
self.status = status
try:
self.reason = RESPONSE_REASONS[self.status][0]
except Exception:
self.reason = 'Fake'
self.body = body
self.resp_headers = HeaderKeyDict()
if headers:
self.resp_headers.update(headers)
self.etag = None
def __call__(self, env, start_response):
method = env["REQUEST_METHOD"]
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env["PATH_INFO"]
_, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True)
if env.get("QUERY_STRING"):
path += "?" + env["QUERY_STRING"]
if "swift.authorize" in env:
resp = env["swift.authorize"](swob.Request(env))
if resp:
return resp(env, start_response)
req_headers = swob.Request(env).headers
self.swift_sources.append(env.get("swift.source"))
self.txn_ids.append(env.get("swift.trans_id"))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses:
resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"])
headers = HeaderKeyDict(raw_headers)
elif method == "HEAD" and ("GET", path) in self._responses:
resp_class, raw_headers, body = self._find_response("GET", path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == "GET" and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % ((method, path),))
self._calls.append((method, path, req_headers))
# simulate object PUT
if method == "PUT" and obj:
input = env["wsgi.input"].read()
etag = md5(input).hexdigest()
headers.setdefault("Etag", etag)
headers.setdefault("Content-Length", len(input))
# keep it for subsequent GET requests later
self.uploaded[path] = (deepcopy(headers), input)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"]
# range requests ought to work, which require conditional_response=True
req = swob.Request(env)
resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD"))
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def getheaders(self):
etag = self.etag
if not etag:
if isinstance(self.body, str):
etag = '"' + md5(self.body).hexdigest() + '"'
else:
etag = '"68b329da9893e34099c7d8ad5cb9c940"'
headers = HeaderKeyDict({
'content-length': len(self.body),
'content-type': 'x-application/test',
'x-timestamp': self.timestamp,
'x-backend-timestamp': self.timestamp,
'last-modified': self.timestamp,
'x-object-meta-test': 'testing',
'x-delete-at': '9876543210',
'etag': etag,
'x-works': 'yes',
})
if self.status // 100 == 2:
headers['x-account-container-count'] = \
kwargs.get('count', 12345)
if not self.timestamp:
# when timestamp is None, HeaderKeyDict raises KeyError
headers.pop('x-timestamp', None)
try:
if next(container_ts_iter) is False:
headers['x-container-timestamp'] = '1'
except StopIteration:
pass
am_slow, value = self.get_slow()
if am_slow:
headers['content-length'] = '4'
headers.update(self.headers)
return headers.items()
class StubResponse(object):
def __init__(self, status, body='', headers=None, frag_index=None):
self.status = status
self.body = body
self.readable = BytesIO(body)
self.headers = HeaderKeyDict(headers)
if frag_index is not None:
self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index
fake_reason = ('Fake', 'This response is a lie.')
self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0]
def getheader(self, header_name, default=None):
return self.headers.get(header_name, default)
def getheaders(self):
if 'Content-Length' not in self.headers:
self.headers['Content-Length'] = len(self.body)
return self.headers.items()
def read(self, amt=0):
return self.readable.read(amt)
def test_direct_head_container(self):
headers = HeaderKeyDict(key='value')
with mocked_http_conn(200, headers) as conn:
resp = direct_client.direct_head_container(self.node, self.part,
self.account,
self.container)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(headers, resp)
def test_obj_put_legacy_updates(self):
ts = (normalize_timestamp(t) for t in
itertools.count(int(time())))
policy = POLICIES.get_by_index(0)
# setup updater
conf = {
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
}
async_dir = os.path.join(self.sda1, get_async_dir(policy))
os.mkdir(async_dir)
account, container, obj = 'a', 'c', 'o'
# write an async
for op in ('PUT', 'DELETE'):
self.logger._clear()
daemon = object_updater.ObjectUpdater(conf, logger=self.logger)
dfmanager = DiskFileManager(conf, daemon.logger)
# don't include storage-policy-index in headers_out pickle
headers_out = HeaderKeyDict({
'x-size': 0,
'x-content-type': 'text/plain',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-timestamp': next(ts),
})
data = {'op': op, 'account': account, 'container': container,
'obj': obj, 'headers': headers_out}
dfmanager.pickle_async_update(self.sda1, account, container, obj,
data, next(ts), policy)
request_log = []
def capture(*args, **kwargs):
request_log.append((args, kwargs))
# run once
fake_status_codes = [200, 200, 200]
with mocked_http_conn(*fake_status_codes, give_connect=capture):
daemon.run_once()
self.assertEqual(len(fake_status_codes), len(request_log))
for request_args, request_kwargs in request_log:
ip, part, method, path, headers, qs, ssl = request_args
self.assertEqual(method, op)
self.assertEqual(headers['X-Backend-Storage-Policy-Index'],
str(int(policy)))
self.assertEqual(daemon.logger.get_increment_counts(),
{'successes': 1, 'unlinks': 1,
'async_pendings': 1})
def test_head_large_object_no_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-no-segments',
environ={'REQUEST_METHOD': 'HEAD'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"], '"%s"' % md5hex(""))
self.assertEqual(headers["Content-Length"], "0")
# one request to HEAD the manifest
# one request for the first page of listings
# *zero* requests for the second page of listings
self.assertEqual(
self.app.calls,
[('HEAD', '/v1/AUTH_test/mancon/manifest-no-segments'),
('GET', '/v1/AUTH_test/c?prefix=noseg_')])
def test_get_undersize_segment(self):
# If we send a Content-Length header to the client, it's based on the
# container listing. If a segment gets smaller by the time we get to
# it (like if a client uploads a smaller segment w/the same name), we
# need to raise an exception so that the connection will be closed by
# the WSGI server. Otherwise, the WSGI server will be waiting for the
# next request, the client will still be waiting for the rest of the
# response, and nobody will be happy.
# Shrink it by a single byte
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
'cccc')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, b'aaaaabbbbbccccdddddeeeee')
def test_direct_put_object_header_content_length(self):
contents = six.StringIO('123456')
stub_headers = HeaderKeyDict({'Content-Length': '6'})
with mocked_http_conn(200) as conn:
resp = direct_client.direct_put_object(self.node,
self.part,
self.account,
self.container,
self.obj,
contents,
headers=stub_headers)
self.assertEqual('PUT', conn.method)
self.assertEqual(conn.req_headers['Content-length'], '6')
self.assertEqual(md5('123456').hexdigest(), resp)
def __init__(self, stype, method, node, part, path, resp, host=None):
# host can be used to override the node ip and port reported in
# the exception
host = host if host is not None else node
full_path = quote('/%s/%s%s' % (node['device'], part, path))
msg = '%s server %s:%s direct %s %r gave status %s' % (
stype, host['ip'], host['port'], method, full_path, resp.status)
headers = HeaderKeyDict(resp.getheaders())
super(DirectClientException, self).__init__(msg,
http_host=host['ip'],
http_port=host['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason,
http_headers=headers)
def test_direct_head_object(self):
headers = HeaderKeyDict({'x-foo': 'bar'})
with mocked_http_conn(200, headers) as conn:
resp = direct_client.direct_head_object(
self.node, self.part, self.account, self.container,
self.obj, headers=headers)
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual('bar', conn.req_headers.get('x-foo'))
self.assertTrue('x-timestamp' not in conn.req_headers,
'x-timestamp was in HEAD request headers')
self.assertEqual(headers, resp)
def test_get_manifest_passthrough(self):
# reregister it with the query param
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest?multipart-manifest=get',
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
'X-Object-Manifest': 'c/seg'},
'manifest-contents')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET',
'QUERY_STRING': 'multipart-manifest=get'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"], "manifest-etag")
self.assertEqual(body, "manifest-contents")
def test_error_fetching_second_segment(self):
self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPForbidden,
{}, None)
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertEqual(''.join(body), "aaaaa") # first segment made it out
self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [
'While processing manifest /v1/AUTH_test/mancon/manifest, '
'got 403 while retrieving /v1/AUTH_test/c/seg_02',
])
def test_error_fetching_second_segment(self):
self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPForbidden,
{}, None)
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body, exc = self.call_dlo(req, expect_exception=True)
headers = HeaderKeyDict(headers)
self.assertTrue(isinstance(exc, exceptions.SegmentError))
self.assertEqual(status, "200 OK")
self.assertEqual(''.join(body), "aaaaa") # first segment made it out
err_lines = self.dlo.logger.get_lines_for_level('error')
self.assertEqual(len(err_lines), 1)
self.assertTrue(err_lines[0].startswith(
'ERROR: An error occurred while retrieving segments'))
def test_account_listing_response(self):
req = Request.blank('')
now = time.time()
with mock.patch('time.time', new=lambda: now):
resp = utils.account_listing_response('a', req, 'text/plain')
self.assertEqual(resp.status_int, 204)
expected = HeaderKeyDict({
'Content-Type': 'text/plain; charset=utf-8',
'X-Account-Container-Count': 0,
'X-Account-Object-Count': 0,
'X-Account-Bytes-Used': 0,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
})
self.assertEqual(expected, resp.headers)
self.assertEqual(b'', resp.body)
def direct_get_object(node, part, account, container, obj, conn_timeout=5,
response_timeout=15, resp_chunk_size=None, headers=None):
"""
Get object directly from the object server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:param resp_chunk_size: if defined, chunk size of data to read.
:param headers: dict to be passed into HTTPConnection headers
:returns: a tuple of (response headers, the object's contents) The response
headers will be a HeaderKeyDict.
:raises ClientException: HTTP GET request failed
"""
if headers is None:
headers = {}
ip, port = get_ip_port(node, headers)
path = _make_path(account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(ip, port, node['device'], part,
'GET', path, headers=gen_headers(headers))
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
resp.read()
raise DirectClientException('Object', 'GET', node, part, path, resp)
if resp_chunk_size:
def _object_body():
buf = resp.read(resp_chunk_size)
while buf:
yield buf
buf = resp.read(resp_chunk_size)
object_body = _object_body()
else:
object_body = resp.read()
resp_headers = HeaderKeyDict()
for header, value in resp.getheaders():
resp_headers[header] = value
return resp_headers, object_body
def _get_direct_account_container(path, stype, node, part,
marker=None, limit=None,
prefix=None, delimiter=None,
conn_timeout=5, response_timeout=15,
end_marker=None, reverse=None, headers=None):
"""Base class for get direct account and container.
Do not use directly use the get_direct_account or
get_direct_container instead.
"""
if headers is None:
headers = {}
params = ['format=json']
if marker:
params.append('marker=%s' % quote(marker))
if limit:
params.append('limit=%d' % limit)
if prefix:
params.append('prefix=%s' % quote(prefix))
if delimiter:
params.append('delimiter=%s' % quote(delimiter))
if end_marker:
params.append('end_marker=%s' % quote(end_marker))
if reverse:
params.append('reverse=%s' % quote(reverse))
qs = '&'.join(params)
ip, port = get_ip_port(node, headers)
with Timeout(conn_timeout):
conn = http_connect(ip, port, node['device'], part,
'GET', path, query_string=qs,
headers=gen_headers(hdrs_in=headers))
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
resp.read()
raise DirectClientException(stype, 'GET', node, part, path, resp)
resp_headers = HeaderKeyDict()
for header, value in resp.getheaders():
resp_headers[header] = value
if resp.status == HTTP_NO_CONTENT:
resp.read()
return resp_headers, []
return resp_headers, json.loads(resp.read())
def _get_direct_account_container(path,
stype,
node,
part,
marker=None,
limit=None,
prefix=None,
delimiter=None,
conn_timeout=5,
response_timeout=15):
"""Base class for get direct account and container.
Do not use directly use the get_direct_account or
get_direct_container instead.
"""
qs = 'format=json'
if marker:
qs += '&marker=%s' % quote(marker)
if limit:
qs += '&limit=%d' % limit
if prefix:
qs += '&prefix=%s' % quote(prefix)
if delimiter:
qs += '&delimiter=%s' % quote(delimiter)
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'GET',
path,
query_string=qs,
headers=gen_headers())
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
resp.read()
raise DirectClientException(stype, 'GET', node, part, path, resp)
resp_headers = HeaderKeyDict()
for header, value in resp.getheaders():
resp_headers[header] = value
if resp.status == HTTP_NO_CONTENT:
resp.read()
return resp_headers, []
return resp_headers, json.loads(resp.read())
def test_set(self):
# mappings = ((<tuple of input vals>, <expected output val>), ...)
mappings = (((1.618, '1.618', b'1.618', u'1.618'),
'1.618'), ((20, '20', b'20', u'20'), '20'),
((True, 'True', b'True', u'True'),
'True'), ((False, 'False', b'False', u'False'), 'False'))
for vals, expected in mappings:
for val in vals:
headers = HeaderKeyDict(test=val)
actual = headers['test']
self.assertEqual(
expected, actual, 'Expected %s but got %s for val %s' %
(expected, actual, val))
self.assertIsInstance(
actual, str,
'Expected type str but got %s for val %s of type %s' %
(type(actual), val, type(val)))
def test_setdefault(self):
headers = HeaderKeyDict()
# it gets set
headers.setdefault("x-rubber-ducky", "the one")
self.assertEqual(headers["X-Rubber-Ducky"], "the one")
# it has the right return value
ret = headers.setdefault("x-boat", "dinghy")
self.assertEqual(ret, "dinghy")
ret = headers.setdefault("x-boat", "yacht")
self.assertEqual(ret, "dinghy")
# shouldn't crash
headers.setdefault("x-sir-not-appearing-in-this-request", None)
def test_setdefault(self):
headers = HeaderKeyDict()
# it gets set
headers.setdefault('x-rubber-ducky', 'the one')
self.assertEqual(headers['X-Rubber-Ducky'], 'the one')
# it has the right return value
ret = headers.setdefault('x-boat', 'dinghy')
self.assertEqual(ret, 'dinghy')
ret = headers.setdefault('x-boat', 'yacht')
self.assertEqual(ret, 'dinghy')
# shouldn't crash
headers.setdefault('x-sir-not-appearing-in-this-request', None)
def test_direct_put_object_header_content_length(self):
contents = io.BytesIO(b'123456')
stub_headers = HeaderKeyDict({'Content-Length': '6'})
with mocked_http_conn(200) as conn:
resp = direct_client.direct_put_object(self.node,
self.part,
self.account,
self.container,
self.obj,
contents,
headers=stub_headers)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual('PUT', conn.method)
self.assertEqual(conn.req_headers['Content-length'], '6')
self.assertEqual(
md5(b'123456', usedforsecurity=False).hexdigest(), resp)
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp.now().internal
headers = HeaderKeyDict(
{'X-Backend-Important-Timestamp': important_timestamp})
with mocked_http_conn(404, headers) as conn:
with self.assertRaises(ClientException) as raised:
direct_client.direct_head_container(self.node, self.part,
self.account,
self.container)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(raised.exception.http_status, 404)
self.assertEqual(raised.exception.http_headers, headers)
def test_direct_head_container_error(self):
headers = HeaderKeyDict(key='value')
with mocked_http_conn(503, headers) as conn:
with self.assertRaises(ClientException) as raised:
direct_client.direct_head_container(self.node, self.part,
self.account,
self.container)
# check request
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(raised.exception.http_status, 503)
self.assertEqual(raised.exception.http_headers, headers)
self.assertTrue('HEAD' in str(raised.exception))
def test_error_fetching_second_segment(self):
self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPForbidden,
{}, None)
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
# first segment made it out
self.assertEqual(body, b'aaaaa')
self.assertEqual(self.app.unread_requests, {})
self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [
'While processing manifest /v1/AUTH_test/mancon/manifest, '
'got 403 (<html><h1>Forbidden</h1><p>Access was denied to this '
'reso...) while retrieving /v1/AUTH_test/c/seg_02',
])
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp(time.time()).internal
headers = HeaderKeyDict({'X-Backend-Important-Timestamp':
important_timestamp})
with mocked_http_conn(404, headers) as conn:
try:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
except Exception as err:
self.assertTrue(isinstance(err, ClientException))
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(err.http_status, 404)
self.assertEqual(err.http_headers, headers)
def test_direct_head_container_replication_net(self):
headers = HeaderKeyDict(key='value')
with mocked_http_conn(200, headers) as conn:
resp = direct_client.direct_head_container(
self.node,
self.part,
self.account,
self.container,
headers={'X-Backend-Use-Replication-Network': 'on'})
self.assertEqual(conn.host, self.node['replication_ip'])
self.assertEqual(conn.port, self.node['replication_port'])
self.assertNotEqual(conn.host, self.node['ip'])
self.assertNotEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(headers, resp)
def test_mismatched_etag_fetching_second_segment(self):
self.app.register(
'GET', '/v1/AUTH_test/c/seg_02',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
'WRONG')
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertEqual(headers['Content-Length'], "25")
# stop after error
self.assertEqual(body, b"aaaaaWRONG")
log_lines = self.dlo.logger.get_lines_for_level('error')
self.assertEqual(len(log_lines), 1,
'Expected one log line, got %r' % log_lines)
self.assertEqual(log_lines[0][:21], 'Bad MD5 checksum for ')
def test_direct_head_container_error(self):
headers = HeaderKeyDict(key='value')
with mocked_http_conn(503, headers) as conn:
try:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
# check request
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(err.http_status, 503)
self.assertEqual(err.http_headers, headers)
self.assertTrue('HEAD' in str(err))
def test_direct_get_container(self):
headers = HeaderKeyDict({'key': 'value'})
body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]'
with mocked_http_conn(200, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container,
marker='marker', prefix='prefix', delimiter='delimiter',
limit=1000)
self.assertEqual(conn.req_headers['user-agent'],
'direct-client %s' % os.getpid())
self.assertEqual(headers, resp_headers)
self.assertEqual(json.loads(body), resp)
self.assertTrue('marker=marker' in conn.query_string)
self.assertTrue('delimiter=delimiter' in conn.query_string)
self.assertTrue('limit=1000' in conn.query_string)
self.assertTrue('prefix=prefix' in conn.query_string)
self.assertTrue('format=json' in conn.query_string)
def accepter(sock, return_code):
try:
with Timeout(3):
inc = sock.makefile('rb')
out = sock.makefile('wb')
out.write(b'HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' %
return_code)
out.flush()
self.assertEqual(inc.readline(),
b'PUT /sda1/0/a/c/o HTTP/1.1\r\n')
headers = HeaderKeyDict()
line = bytes_to_wsgi(inc.readline())
while line and line != '\r\n':
headers[line.split(':')[0]] = \
line.split(':')[1].strip()
line = bytes_to_wsgi(inc.readline())
self.assertIn('x-container-timestamp', headers)
self.assertIn('X-Backend-Storage-Policy-Index', headers)
except BaseException as err:
return err
return None
def test_mismatched_length_fetching_second_segment(self):
self.app.register(
'GET', '/v1/AUTH_test/c/seg_02',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbb")},
# Use a list so we can get a discrepency between content-length and
# number of bytes in the app_iter
[b'b' * 4])
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertEqual(headers['Content-Length'], "25")
# stop after error
self.assertEqual(body, b"aaaaabbbb")
log_lines = self.dlo.logger.get_lines_for_level('error')
self.assertEqual(len(log_lines), 1,
'Expected one log line, got %r' % log_lines)
self.assertEqual(log_lines[0][:24], 'Bad response length for ')
def test_etag_comparison_ignores_quotes(self):
# a little future-proofing here in case we ever fix this in swob
self.app.register(
'HEAD', '/v1/AUTH_test/mani/festo',
swob.HTTPOk, {'Content-Length': '0', 'Etag': 'blah',
'X-Object-Manifest': 'c/quotetags'}, None)
self.app.register(
'GET', '/v1/AUTH_test/c?prefix=quotetags',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps([{"hash": "\"abc\"", "bytes": 5, "name": "quotetags1",
"last_modified": "2013-11-22T02:42:14.261620",
"content-type": "application/octet-stream"},
{"hash": "def", "bytes": 5, "name": "quotetags2",
"last_modified": "2013-11-22T02:42:14.261620",
"content-type": "application/octet-stream"}]))
req = swob.Request.blank('/v1/AUTH_test/mani/festo',
environ={'REQUEST_METHOD': 'HEAD'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"],
'"' + hashlib.md5("abcdef").hexdigest() + '"')
def direct_head_container(node, part, account, container, conn_timeout=5,
response_timeout=15):
"""
Request container information directly from the container server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a dict containing the response's headers in a HeaderKeyDict
:raises ClientException: HTTP HEAD request failed
"""
path = '/%s/%s' % (account, container)
resp = _make_req(node, part, 'HEAD', path, gen_headers(),
'Container', conn_timeout, response_timeout)
resp_headers = HeaderKeyDict()
for header, value in resp.getheaders():
resp_headers[header] = value
return resp_headers
def resolve_etag_is_at_header(req, metadata):
"""
Helper function to resolve an alternative etag value that may be stored in
metadata under an alternate name.
The value of the request's X-Backend-Etag-Is-At header (if it exists) is a
comma separated list of alternate names in the metadata at which an
alternate etag value may be found. This list is processed in order until an
alternate etag is found.
The left most value in X-Backend-Etag-Is-At will have been set by the left
most middleware, or if no middleware, by ECObjectController, if an EC
policy is in use. The left most middleware is assumed to be the authority
on what the etag value of the object content is.
The resolver will work from left to right in the list until it finds a
value that is a name in the given metadata. So the left most wins, IF it
exists in the metadata.
By way of example, assume the encrypter middleware is installed. If an
object is *not* encrypted then the resolver will not find the encrypter
middleware's alternate etag sysmeta (X-Object-Sysmeta-Crypto-Etag) but will
then find the EC alternate etag (if EC policy). But if the object *is*
encrypted then X-Object-Sysmeta-Crypto-Etag is found and used, which is
correct because it should be preferred over X-Object-Sysmeta-Ec-Etag.
:param req: a swob Request
:param metadata: a dict containing object metadata
:return: an alternate etag value if any is found, otherwise None
"""
alternate_etag = None
metadata = HeaderKeyDict(metadata)
if "X-Backend-Etag-Is-At" in req.headers:
names = list_from_csv(req.headers["X-Backend-Etag-Is-At"])
for name in names:
if name in metadata:
alternate_etag = metadata[name]
break
return alternate_etag
def test_get_manifest(self):
expected_etag = '"%s"' % md5hex(
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
md5hex("ddddd") + md5hex("eeeee"))
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"], expected_etag)
self.assertEqual(headers["Content-Length"], "25")
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
for _, _, hdrs in self.app.calls_with_headers[1:]:
ua = hdrs.get("User-Agent", "")
self.assertTrue("DLO MultipartGET" in ua)
self.assertFalse("DLO MultipartGET DLO MultipartGET" in ua)
# the first request goes through unaltered
self.assertFalse(
"DLO MultipartGET" in self.app.calls_with_headers[0][2])
# we set swift.source for everything but the first request
self.assertEqual(self.app.swift_sources,
[None, 'DLO', 'DLO', 'DLO', 'DLO', 'DLO', 'DLO'])
def getheaders(self):
etag = self.etag
if not etag:
if isinstance(self.body, str):
etag = '"' + md5(self.body).hexdigest() + '"'
else:
etag = '"68b329da9893e34099c7d8ad5cb9c940"'
headers = HeaderKeyDict(
{
"content-length": len(self.body),
"content-type": "x-application/test",
"x-timestamp": self.timestamp,
"x-backend-timestamp": self.timestamp,
"last-modified": self.timestamp,
"x-object-meta-test": "testing",
"x-delete-at": "9876543210",
"etag": etag,
"x-works": "yes",
}
)
if self.status // 100 == 2:
headers["x-account-container-count"] = kwargs.get("count", 12345)
if not self.timestamp:
# when timestamp is None, HeaderKeyDict raises KeyError
headers.pop("x-timestamp", None)
try:
if next(container_ts_iter) is False:
headers["x-container-timestamp"] = "1"
except StopIteration:
pass
am_slow, value = self.get_slow()
if am_slow:
headers["content-length"] = "4"
headers.update(self.headers)
return headers.items()
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env['PATH_INFO']
_, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
rest_with_last=True)
if env.get('QUERY_STRING'):
path += '?' + env['QUERY_STRING']
if 'swift.authorize' in env:
resp = env['swift.authorize'](swob.Request(env))
if resp:
return resp(env, start_response)
req = swob.Request(env)
self.swift_sources.append(env.get('swift.source'))
self.txn_ids.append(env.get('swift.trans_id'))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if (env.get('QUERY_STRING')
and (method, env['PATH_INFO']) in self._responses):
resp_class, raw_headers, body = self._find_response(
method, env['PATH_INFO'])
headers = HeaderKeyDict(raw_headers)
elif method == 'HEAD' and ('GET', path) in self._responses:
resp_class, raw_headers, body = self._find_response(
'GET', path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == 'GET' and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % (
(method, path),))
# simulate object PUT
if method == 'PUT' and obj:
put_body = ''.join(iter(env['wsgi.input'].read, ''))
if 'swift.callback.update_footers' in env:
footers = HeaderKeyDict()
env['swift.callback.update_footers'](footers)
req.headers.update(footers)
etag = md5(put_body).hexdigest()
headers.setdefault('Etag', etag)
headers.setdefault('Content-Length', len(put_body))
# keep it for subsequent GET requests later
self.uploaded[path] = (dict(req.headers), put_body)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
# simulate object POST
elif method == 'POST' and obj:
metadata, data = self.uploaded.get(path, ({}, None))
# select items to keep from existing...
new_metadata = dict(
(k, v) for k, v in metadata.items()
if (not is_user_meta('object', k) and not
is_object_transient_sysmeta(k)))
# apply from new
new_metadata.update(
dict((k, v) for k, v in req.headers.items()
if (is_user_meta('object', k) or
is_object_transient_sysmeta(k) or
k.lower == 'content-type')))
self.uploaded[path] = new_metadata, data
# note: tests may assume this copy of req_headers is case insensitive
# so we deliberately use a HeaderKeyDict
self._calls.append(
FakeSwiftCall(method, path, HeaderKeyDict(req.headers)))
backend_etag_header = req.headers.get('X-Backend-Etag-Is-At')
conditional_etag = None
if backend_etag_header and backend_etag_header in headers:
# Apply conditional etag overrides
conditional_etag = headers[backend_etag_header]
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
resp = resp_class(
req=req, headers=headers, app_iter=body,
conditional_response=req.method in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
else:
resp = resp_class(
req=req, headers=headers, body=body,
conditional_response=req.method in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def __call__(self, env, start_response):
method = env["REQUEST_METHOD"]
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env["PATH_INFO"]
_, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True)
if env.get("QUERY_STRING"):
path += "?" + env["QUERY_STRING"]
if "swift.authorize" in env:
resp = env["swift.authorize"](swob.Request(env))
if resp:
return resp(env, start_response)
req = swob.Request(env)
self.swift_sources.append(env.get("swift.source"))
self.txn_ids.append(env.get("swift.trans_id"))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses:
resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"])
headers = HeaderKeyDict(raw_headers)
elif method == "HEAD" and ("GET", path) in self._responses:
resp_class, raw_headers, body = self._find_response("GET", path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == "GET" and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % ((method, path),))
# simulate object PUT
if method == "PUT" and obj:
put_body = "".join(iter(env["wsgi.input"].read, ""))
if "swift.callback.update_footers" in env:
footers = HeaderKeyDict()
env["swift.callback.update_footers"](footers)
req.headers.update(footers)
etag = md5(put_body).hexdigest()
headers.setdefault("Etag", etag)
headers.setdefault("Content-Length", len(put_body))
# keep it for subsequent GET requests later
self.uploaded[path] = (dict(req.headers), put_body)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"]
# simulate object POST
elif method == "POST" and obj:
metadata, data = self.uploaded.get(path, ({}, None))
# select items to keep from existing...
new_metadata = dict(
(k, v)
for k, v in metadata.items()
if (not is_user_meta("object", k) and not is_object_transient_sysmeta(k))
)
# apply from new
new_metadata.update(
dict(
(k, v)
for k, v in req.headers.items()
if (is_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower == "content-type")
)
)
self.uploaded[path] = new_metadata, data
# note: tests may assume this copy of req_headers is case insensitive
# so we deliberately use a HeaderKeyDict
self._calls.append((method, path, HeaderKeyDict(req.headers)))
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
resp = resp_class(
req=req, headers=headers, app_iter=body, conditional_response=req.method in ("GET", "HEAD")
)
else:
resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD"))
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def test_keys(self):
headers = HeaderKeyDict()
headers["content-length"] = 20
headers["cOnTent-tYpe"] = "text/plain"
headers["SomeThing-eLse"] = "somevalue"
self.assertEqual(set(headers.keys()), set(("Content-Length", "Content-Type", "Something-Else")))
def test_update(self):
headers = HeaderKeyDict()
headers.update({'Content-Length': '0'})
headers.update([('Content-Type', 'text/plain')])
self.assertEqual(headers['Content-Length'], '0')
self.assertEqual(headers['Content-Type'], 'text/plain')
def test_get(self):
headers = HeaderKeyDict()
headers['content-length'] = 20
self.assertEqual(headers.get('CONTENT-LENGTH'), '20')
self.assertEqual(headers.get('something-else'), None)
self.assertEqual(headers.get('something-else', True), True)
