def test_get_suffix_range_many_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=-5'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_suffix_range_many_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=-5'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_multi_range(self):
# DLO doesn't support multi-range GETs. The way that you express that
# in HTTP is to return a 200 response containing the whole entity.
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=5-9,15-19'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def test_get_multi_range(self):
# DLO doesn't support multi-range GETs. The way that you express that
# in HTTP is to return a 200 response containing the whole entity.
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=5-9,15-19'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def test_get_undersize_segment(self):
# If we send a Content-Length header to the client, it's based on the
# container listing. If a segment gets smaller by the time we get to
# it (like if a client uploads a smaller segment w/the same name), we
# need to raise an exception so that the connection will be closed by
# the WSGI server. Otherwise, the WSGI server will be waiting for the
# next request, the client will still be waiting for the rest of the
# response, and nobody will be happy.
# Shrink it by a single byte
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
'cccc')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body, exc = self.call_dlo(req, expect_exception=True)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee')
self.assertTrue(isinstance(exc, exceptions.SegmentError))
def test_get_big_manifest(self):
self.app.register(
'GET', '/v1/AUTH_test/mancon/big-manifest', swob.HTTPOk, {
'Content-Length': '17000',
'Etag': 'manifest-etag',
'X-Object-Manifest': 'c/seg'
}, b'manifest-contents' * 1000)
req = swob.Request.blank('/v1/AUTH_test/mancon/big-manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertEqual(headers["Content-Length"], "25")
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
expected_etag = '"%s"' % md5hex(
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
md5hex("ddddd") + md5hex("eeeee"))
self.assertEqual(headers.get("Etag"), expected_etag)
self.assertEqual(
self.app.unread_requests,
{
# Since we don't know how big this will be, we just disconnect
('GET', '/v1/AUTH_test/mancon/big-manifest'):
1,
})
def test_get_undersize_segment(self):
# If we send a Content-Length header to the client, it's based on the
# container listing. If a segment gets smaller by the time we get to
# it (like if a client uploads a smaller segment w/the same name), we
# need to raise an exception so that the connection will be closed by
# the WSGI server. Otherwise, the WSGI server will be waiting for the
# next request, the client will still be waiting for the rest of the
# response, and nobody will be happy.
# Shrink it by a single byte
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
'cccc')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body, exc = self.call_dlo(req, expect_exception=True)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee')
self.assertTrue(isinstance(exc, exceptions.SegmentError))
def test_get_oversize_segment(self):
# If we send a Content-Length header to the client, it's based on the
# container listing. If a segment gets bigger by the time we get to it
# (like if a client uploads a bigger segment w/the same name), we need
# to not send anything beyond the length we promised. Also, we should
# probably raise an exception.
# This is now longer than the original seg_03+seg_04+seg_05 combined
self.app.register('GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {
'Content-Length': '20',
'Etag': 'seg03-etag'
}, 'cccccccccccccccccccc')
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
self.assertEqual(
self.app.calls,
[('GET', '/v1/AUTH_test/mancon/manifest'),
('GET', '/v1/AUTH_test/c?prefix=seg'),
('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])
class StubResponse(object):
def __init__(self, status, body=b'', headers=None, frag_index=None):
self.status = status
self.body = body
self.readable = BytesIO(body)
self.headers = HeaderKeyDict(headers)
if frag_index is not None:
self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index
fake_reason = ('Fake', 'This response is a lie.')
self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0]
def getheader(self, header_name, default=None):
return self.headers.get(header_name, default)
def getheaders(self):
if 'Content-Length' not in self.headers:
self.headers['Content-Length'] = len(self.body)
return self.headers.items()
def read(self, amt=0):
return self.readable.read(amt)
def __repr__(self):
info = ['Status: %s' % self.status]
if self.headers:
info.append('Headers: %r' % dict(self.headers))
if self.body:
info.append('Body: %r' % self.body)
return '<StubResponse %s>' % ', '.join(info)
def test_get_oversize_segment(self):
# If we send a Content-Length header to the client, it's based on the
# container listing. If a segment gets bigger by the time we get to it
# (like if a client uploads a bigger segment w/the same name), we need
# to not send anything beyond the length we promised. Also, we should
# probably raise an exception.
# This is now longer than the original seg_03+seg_04+seg_05 combined
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '20', 'Etag': 'seg03-etag'},
'cccccccccccccccccccc')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
self.assertEqual(
self.app.calls,
[('GET', '/v1/AUTH_test/mancon/manifest'),
('GET', '/v1/AUTH_test/c?prefix=seg'),
('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])
def test_head_large_object_too_many_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'HEAD'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
# etag is manifest's etag
self.assertEqual(headers["Etag"], "etag-manyseg")
self.assertIsNone(headers.get("Content-Length"))
def test_head_large_object_too_many_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'HEAD'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
# etag is manifest's etag
self.assertEqual(headers["Etag"], "etag-manyseg")
self.assertIsNone(headers.get("Content-Length"))
class StubResponse(object):
def __init__(self,
status,
body=b'',
headers=None,
frag_index=None,
slowdown=None):
self.status = status
self.body = body
self.readable = BytesIO(body)
try:
self._slowdown = iter(slowdown)
except TypeError:
self._slowdown = iter([slowdown])
self.headers = HeaderKeyDict(headers)
if frag_index is not None:
self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index
fake_reason = ('Fake', 'This response is a lie.')
self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0]
def slowdown(self):
try:
wait = next(self._slowdown)
except StopIteration:
wait = None
if wait is not None:
eventlet.sleep(wait)
def nuke_from_orbit(self):
if hasattr(self, 'swift_conn'):
self.swift_conn.close()
def getheader(self, header_name, default=None):
return self.headers.get(header_name, default)
def getheaders(self):
if 'Content-Length' not in self.headers:
self.headers['Content-Length'] = len(self.body)
return self.headers.items()
def read(self, amt=0):
self.slowdown()
return self.readable.read(amt)
def readline(self, size=-1):
self.slowdown()
return self.readable.readline(size)
def __repr__(self):
info = ['Status: %s' % self.status]
if self.headers:
info.append('Headers: %r' % dict(self.headers))
if self.body:
info.append('Body: %r' % self.body)
return '<StubResponse %s>' % ', '.join(info)
def test_get_range_many_segments_satisfiability_unknown(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=10-22'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
# this requires multiple pages of container listing, so we can't send
# a Content-Length header
self.assertIsNone(headers.get("Content-Length"))
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_range_many_segments_satisfiability_unknown(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=10-22'})
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
# this requires multiple pages of container listing, so we can't send
# a Content-Length header
self.assertIsNone(headers.get("Content-Length"))
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def _validate_etag_and_update_sysmeta(self, req, symlink_target_path,
etag):
if req.environ.get('swift.symlink_override'):
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \
req.headers[TGT_BYTES_SYMLINK_HDR]
return
# next we'll make sure the E-Tag matches a real object
new_req = make_subrequest(req.environ,
path=wsgi_quote(symlink_target_path),
method='HEAD',
swift_source='SYM')
if req.allow_reserved_names:
new_req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
self._last_target_path = symlink_target_path
resp = self._recursive_get_head(new_req,
target_etag=etag,
follow_softlinks=False)
if self._get_status_int() == HTTP_NOT_FOUND:
raise HTTPConflict(body='X-Symlink-Target does not exist',
request=req,
headers={
'Content-Type': 'text/plain',
'Content-Location': self._last_target_path
})
if not is_success(self._get_status_int()):
drain_and_close(resp)
raise status_map[self._get_status_int()](request=req)
response_headers = HeaderKeyDict(self._response_headers)
# carry forward any etag update params (e.g. "slo_etag"), we'll append
# symlink_target_* params to this header after this method returns
override_header = get_container_update_override_key('etag')
if override_header in response_headers and \
override_header not in req.headers:
sep, params = response_headers[override_header].partition(';')[1:]
req.headers[override_header] = MD5_OF_EMPTY_STRING + sep + params
# It's troublesome that there's so much leakage with SLO
if 'X-Object-Sysmeta-Slo-Etag' in response_headers and \
override_header not in req.headers:
req.headers[override_header] = '%s; slo_etag=%s' % (
MD5_OF_EMPTY_STRING,
response_headers['X-Object-Sysmeta-Slo-Etag'])
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = (
response_headers.get('x-object-sysmeta-slo-size')
or response_headers['Content-Length'])
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
if not req.headers.get('Content-Type'):
req.headers['Content-Type'] = response_headers['Content-Type']
def test_get_range(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=8-17'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "10")
self.assertEqual(body, b'bbcccccddd')
expected_etag = '"%s"' % md5hex(
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
md5hex("ddddd") + md5hex("eeeee"))
self.assertEqual(headers.get("Etag"), expected_etag)
def test_get_range(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=8-17'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "10")
self.assertEqual(body, b'bbcccccddd')
expected_etag = '"%s"' % md5hex(
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
md5hex("ddddd") + md5hex("eeeee"))
self.assertEqual(headers.get("Etag"), expected_etag)
def handle(self, req, start_response):
app_resp = self._app_call(req.environ)
if is_success(self._get_status_int()):
# only decrypt body of 2xx responses
headers = HeaderKeyDict(self._response_headers)
content_type = headers.get('content-type', '').split(';', 1)[0]
if content_type == 'application/json':
app_resp = self.process_json_resp(req, app_resp)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return app_resp
def handle(self, req, start_response):
app_resp = self._app_call(req.environ)
if is_success(self._get_status_int()):
# only decrypt body of 2xx responses
headers = HeaderKeyDict(self._response_headers)
content_type = headers.get('content-type', '').split(';', 1)[0]
if content_type == 'application/json':
app_resp = self.process_json_resp(req, app_resp)
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return app_resp
def test_get_undersize_segment_range(self):
# Shrink it by a single byte
self.app.register('GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {
'Content-Length': '4',
'Etag': md5hex("cccc")
}, 'cccc')
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=0-14'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '206 Partial Content') # sanity check
self.assertEqual(headers.get('Content-Length'), '15') # sanity check
self.assertEqual(body, b'aaaaabbbbbcccc')
class FakeConn(object):
def __init__(self, status, headers=None, body='', **kwargs):
self.status = status
try:
self.reason = RESPONSE_REASONS[self.status][0]
except Exception:
self.reason = 'Fake'
self.body = body
self.resp_headers = HeaderKeyDict()
if headers:
self.resp_headers.update(headers)
self.etag = None
def _update_raw_call_args(self, *args, **kwargs):
capture_attrs = ('host', 'port', 'method', 'path', 'req_headers',
'query_string')
for attr, value in zip(capture_attrs, args[:len(capture_attrs)]):
setattr(self, attr, value)
return self
def getresponse(self):
if self.etag:
self.resp_headers['etag'] = str(self.etag.hexdigest())
if isinstance(self.status, Exception):
raise self.status
return self
def getheader(self, header, default=None):
return self.resp_headers.get(header, default)
def getheaders(self):
return self.resp_headers.items()
def read(self, amt=None):
if isinstance(self.body, six.StringIO):
return self.body.read(amt)
elif amt is None:
return self.body
else:
return Exception('Not a StringIO entry')
def send(self, data):
if not self.etag:
self.etag = md5()
self.etag.update(data)
def test_get_undersize_segment_range(self):
# Shrink it by a single byte
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
'cccc')
req = swob.Request.blank(
'/v1/AUTH_test/mancon/manifest',
environ={'REQUEST_METHOD': 'GET'},
headers={'Range': 'bytes=0-14'})
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(status, '206 Partial Content') # sanity check
self.assertEqual(headers.get('Content-Length'), '15') # sanity check
self.assertEqual(body, b'aaaaabbbbbcccc')
class StubResponse(object):
def __init__(self, status, body='', headers=None, frag_index=None):
self.status = status
self.body = body
self.readable = BytesIO(body)
self.headers = HeaderKeyDict(headers)
if frag_index is not None:
self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index
fake_reason = ('Fake', 'This response is a lie.')
self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0]
def getheader(self, header_name, default=None):
return self.headers.get(header_name, default)
def getheaders(self):
if 'Content-Length' not in self.headers:
self.headers['Content-Length'] = len(self.body)
return self.headers.items()
def read(self, amt=0):
return self.readable.read(amt)
def test_extract_metadata(self):
self.app.register('HEAD', '/v1/a/c?extract-archive=tar', HTTPNoContent,
{}, None)
self.app.register('PUT', '/v1/a/c/obj1?extract-archive=tar',
HTTPCreated, {}, None)
self.app.register('PUT', '/v1/a/c/obj2?extract-archive=tar',
HTTPCreated, {}, None)
# It's a real pain to instantiate TarInfo objects directly; they
# really want to come from a file on disk or a tarball. So, we write
# out some files and add pax headers to them as they get placed into
# the tarball.
with open(os.path.join(self.testdir, "obj1"), "w") as fh1:
fh1.write("obj1 contents\n")
with open(os.path.join(self.testdir, "obj2"), "w") as fh2:
fh2.write("obj2 contents\n")
tar_ball = BytesIO()
tar_file = tarfile.TarFile.open(fileobj=tar_ball,
mode="w",
format=tarfile.PAX_FORMAT)
# With GNU tar 1.27.1 or later (possibly 1.27 as well), a file with
# extended attribute user.thingy = dingy gets put into the tarfile
# with pax_headers containing key/value pair
# (SCHILY.xattr.user.thingy, dingy), both unicode strings (py2: type
# unicode, not type str).
#
# With BSD tar (libarchive), you get key/value pair
# (LIBARCHIVE.xattr.user.thingy, dingy), which strikes me as
# gratuitous incompatibility.
#
# Still, we'll support uploads with both. Just heap more code on the
# problem until you can forget it's under there.
with open(os.path.join(self.testdir, "obj1")) as fh1:
tar_info1 = tar_file.gettarinfo(fileobj=fh1, arcname="obj1")
tar_info1.pax_headers[u'SCHILY.xattr.user.mime_type'] = \
u'application/food-diary'
tar_info1.pax_headers[u'SCHILY.xattr.user.meta.lunch'] = \
u'sopa de albóndigas'
tar_info1.pax_headers[
u'SCHILY.xattr.user.meta.afternoon-snack'] = \
u'gigantic bucket of coffee'
tar_file.addfile(tar_info1, fh1)
with open(os.path.join(self.testdir, "obj2")) as fh2:
tar_info2 = tar_file.gettarinfo(fileobj=fh2, arcname="obj2")
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.meta.muppet'] = u'bert'
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.meta.cat'] = u'fluffy'
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.notmeta'] = u'skipped'
tar_file.addfile(tar_info2, fh2)
tar_ball.seek(0)
req = Request.blank('/v1/a/c?extract-archive=tar')
req.environ['REQUEST_METHOD'] = 'PUT'
req.environ['wsgi.input'] = tar_ball
req.headers['transfer-encoding'] = 'chunked'
req.headers['accept'] = 'application/json;q=1.0'
resp = req.get_response(self.bulk)
self.assertEqual(resp.status_int, 200)
# sanity check to make sure the upload worked
upload_status = utils.json.loads(resp.body)
self.assertEqual(upload_status['Number Files Created'], 2)
put1_headers = HeaderKeyDict(self.app.calls_with_headers[1][2])
self.assertEqual(put1_headers.get('Content-Type'),
'application/food-diary')
self.assertEqual(put1_headers.get('X-Object-Meta-Lunch'),
'sopa de alb\xc3\xb3ndigas')
self.assertEqual(put1_headers.get('X-Object-Meta-Afternoon-Snack'),
'gigantic bucket of coffee')
put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2])
self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert')
self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy')
self.assertEqual(put2_headers.get('Content-Type'), None)
self.assertEqual(put2_headers.get('X-Object-Meta-Blah'), None)
def test_get(self):
headers = HeaderKeyDict()
headers['content-length'] = 20
self.assertEqual(headers.get('CONTENT-LENGTH'), '20')
self.assertIsNone(headers.get('something-else'))
self.assertEqual(headers.get('something-else', True), True)
def test_extract_metadata(self):
self.app.register('HEAD', '/v1/a/c?extract-archive=tar',
HTTPNoContent, {}, None)
self.app.register('PUT', '/v1/a/c/obj1?extract-archive=tar',
HTTPCreated, {}, None)
self.app.register('PUT', '/v1/a/c/obj2?extract-archive=tar',
HTTPCreated, {}, None)
# It's a real pain to instantiate TarInfo objects directly; they
# really want to come from a file on disk or a tarball. So, we write
# out some files and add pax headers to them as they get placed into
# the tarball.
with open(os.path.join(self.testdir, "obj1"), "w") as fh1:
fh1.write("obj1 contents\n")
with open(os.path.join(self.testdir, "obj2"), "w") as fh2:
fh2.write("obj2 contents\n")
tar_ball = BytesIO()
tar_file = tarfile.TarFile.open(fileobj=tar_ball, mode="w",
format=tarfile.PAX_FORMAT)
# With GNU tar 1.27.1 or later (possibly 1.27 as well), a file with
# extended attribute user.thingy = dingy gets put into the tarfile
# with pax_headers containing key/value pair
# (SCHILY.xattr.user.thingy, dingy), both unicode strings (py2: type
# unicode, not type str).
#
# With BSD tar (libarchive), you get key/value pair
# (LIBARCHIVE.xattr.user.thingy, dingy), which strikes me as
# gratuitous incompatibility.
#
# Still, we'll support uploads with both. Just heap more code on the
# problem until you can forget it's under there.
with open(os.path.join(self.testdir, "obj1")) as fh1:
tar_info1 = tar_file.gettarinfo(fileobj=fh1,
arcname="obj1")
tar_info1.pax_headers[u'SCHILY.xattr.user.mime_type'] = \
u'application/food-diary'
tar_info1.pax_headers[u'SCHILY.xattr.user.meta.lunch'] = \
u'sopa de albóndigas'
tar_info1.pax_headers[
u'SCHILY.xattr.user.meta.afternoon-snack'] = \
u'gigantic bucket of coffee'
tar_file.addfile(tar_info1, fh1)
with open(os.path.join(self.testdir, "obj2")) as fh2:
tar_info2 = tar_file.gettarinfo(fileobj=fh2,
arcname="obj2")
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.meta.muppet'] = u'bert'
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.meta.cat'] = u'fluffy'
tar_info2.pax_headers[
u'LIBARCHIVE.xattr.user.notmeta'] = u'skipped'
tar_file.addfile(tar_info2, fh2)
tar_ball.seek(0)
req = Request.blank('/v1/a/c?extract-archive=tar')
req.environ['REQUEST_METHOD'] = 'PUT'
req.environ['wsgi.input'] = tar_ball
req.headers['transfer-encoding'] = 'chunked'
req.headers['accept'] = 'application/json;q=1.0'
resp = req.get_response(self.bulk)
self.assertEqual(resp.status_int, 200)
# sanity check to make sure the upload worked
upload_status = utils.json.loads(resp.body)
self.assertEqual(upload_status['Number Files Created'], 2)
put1_headers = HeaderKeyDict(self.app.calls_with_headers[1][2])
self.assertEqual(
put1_headers.get('Content-Type'),
'application/food-diary')
self.assertEqual(
put1_headers.get('X-Object-Meta-Lunch'),
'sopa de alb\xc3\xb3ndigas')
self.assertEqual(
put1_headers.get('X-Object-Meta-Afternoon-Snack'),
'gigantic bucket of coffee')
put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2])
self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert')
self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy')
self.assertEqual(put2_headers.get('Content-Type'), None)
self.assertEqual(put2_headers.get('X-Object-Meta-Blah'), None)
def test_get(self):
headers = HeaderKeyDict()
headers["content-length"] = 20
self.assertEqual(headers.get("CONTENT-LENGTH"), "20")
self.assertEqual(headers.get("something-else"), None)
self.assertEqual(headers.get("something-else", True), True)
def test_get(self):
headers = HeaderKeyDict()
headers['content-length'] = 20
self.assertEqual(headers.get('CONTENT-LENGTH'), '20')
self.assertEqual(headers.get('something-else'), None)
self.assertEqual(headers.get('something-else', True), True)
