def test_206_multiple_ranges(self): fr = FakeResponse( 206, {'Content-Type': 'multipart/byteranges; boundary=asdfasdfasdf'}, (b"--asdfasdfasdf\r\n" b"Content-Type: application/lunch\r\n" b"Content-Range: bytes 0-3/10\r\n" b"\r\n" b"sand\r\n" b"--asdfasdfasdf\r\n" b"Content-Type: application/lunch\r\n" b"Content-Range: bytes 6-9/10\r\n" b"\r\n" b"ches\r\n" b"--asdfasdfasdf--")) doc_iters = rh.http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 0) self.assertEqual(last_byte, 3) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), b'sand') first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 6) self.assertEqual(last_byte, 9) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), b'ches') self.assertRaises(StopIteration, next, doc_iters)
def test_200(self): fr = FakeResponse(200, { 'Content-Length': '10', 'Content-Type': 'application/lunch' }, b'sandwiches') doc_iters = rh.http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 0) self.assertEqual(last_byte, 9) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Length'), '10') self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), b'sandwiches') self.assertRaises(StopIteration, next, doc_iters) fr = FakeResponse(200, { 'Transfer-Encoding': 'chunked', 'Content-Type': 'application/lunch' }, b'sandwiches') doc_iters = rh.http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 0) self.assertIsNone(last_byte) self.assertIsNone(length) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Transfer-Encoding'), 'chunked') self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), b'sandwiches') self.assertRaises(StopIteration, next, doc_iters)
def test_206_multiple_ranges(self): fr = FakeResponse( 206, {'Content-Type': 'multipart/byteranges; boundary=asdfasdfasdf'}, ("--asdfasdfasdf\r\n" "Content-Type: application/lunch\r\n" "Content-Range: bytes 0-3/10\r\n" "\r\n" "sand\r\n" "--asdfasdfasdf\r\n" "Content-Type: application/lunch\r\n" "Content-Range: bytes 6-9/10\r\n" "\r\n" "ches\r\n" "--asdfasdfasdf--")) doc_iters = http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 0) self.assertEqual(last_byte, 3) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), 'sand') first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 6) self.assertEqual(last_byte, 9) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), 'ches') self.assertRaises(StopIteration, next, doc_iters)
def cookie_resp(status, response_headers, exc_info=None): resp_headers = HeaderKeyDict(response_headers) if 'x-auth-token' in resp_headers: auth_token = resp_headers['x-auth-token'] expires_in = int(resp_headers.get('x-auth-token-expires', 0)) storage_url = resp_headers.get('x-storage-url', '') path_parts = urlparse(storage_url) domain = path_parts.netloc secure = False if path_parts.scheme == 'https': secure = True if auth_token and domain: new_cookie = create_auth_cookie('session', domain, token=auth_token, expires_in=expires_in, secure=secure, httponly=True) response_headers.append(('Set-Cookie', new_cookie)) new_cookie = create_auth_cookie('storage', domain, token=storage_url, expires_in=expires_in, secure=secure) response_headers.append(('Set-Cookie', new_cookie)) return start_response(status, response_headers, exc_info)
def test_200(self): fr = FakeResponse( 200, {'Content-Length': '10', 'Content-Type': 'application/lunch'}, 'sandwiches') doc_iters = http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 0) self.assertEqual(last_byte, 9) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Length'), '10') self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), 'sandwiches') self.assertRaises(StopIteration, next, doc_iters) fr = FakeResponse( 200, {'Transfer-Encoding': 'chunked', 'Content-Type': 'application/lunch'}, 'sandwiches') doc_iters = http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 0) self.assertIsNone(last_byte) self.assertIsNone(length) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Transfer-Encoding'), 'chunked') self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), 'sandwiches') self.assertRaises(StopIteration, next, doc_iters)
def cookie_resp(status, response_headers, exc_info=None): resp_headers = HeaderKeyDict(response_headers) if 'x-auth-token' in resp_headers: auth_token = resp_headers['x-auth-token'] expires_in = int(resp_headers.get('x-auth-token-expires', 0)) storage_url = resp_headers.get('x-storage-url', '') path_parts = urlparse(storage_url) domain = path_parts.hostname secure = False if path_parts.scheme == 'https': secure = True if auth_token and domain: new_cookie = create_auth_cookie('session', domain, token=auth_token, expires_in=expires_in, secure=secure, httponly=True) response_headers.append(('Set-Cookie', new_cookie)) new_cookie = create_auth_cookie('storage', domain, token=quote(storage_url, safe=''), expires_in=expires_in, secure=secure) response_headers.append(('Set-Cookie', new_cookie)) return start_response(status, response_headers, exc_info)
def test_206_single_range(self): fr = FakeResponse( 206, { 'Content-Length': '8', 'Content-Type': 'application/lunch', 'Content-Range': 'bytes 1-8/10' }, b'andwiche') doc_iters = rh.http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 1) self.assertEqual(last_byte, 8) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Length'), '8') self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), b'andwiche') self.assertRaises(StopIteration, next, doc_iters) # Chunked response should be treated in the same way as non-chunked one fr = FakeResponse( 206, { 'Transfer-Encoding': 'chunked', 'Content-Type': 'application/lunch', 'Content-Range': 'bytes 1-8/10' }, b'andwiche') doc_iters = rh.http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 1) self.assertEqual(last_byte, 8) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), b'andwiche') self.assertRaises(StopIteration, next, doc_iters)
def test_206_single_range(self): fr = FakeResponse( 206, {'Content-Length': '8', 'Content-Type': 'application/lunch', 'Content-Range': 'bytes 1-8/10'}, 'andwiche') doc_iters = http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 1) self.assertEqual(last_byte, 8) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Length'), '8') self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), 'andwiche') self.assertRaises(StopIteration, next, doc_iters) # Chunked response should be treated in the same way as non-chunked one fr = FakeResponse( 206, {'Transfer-Encoding': 'chunked', 'Content-Type': 'application/lunch', 'Content-Range': 'bytes 1-8/10'}, 'andwiche') doc_iters = http_response_to_document_iters(fr) first_byte, last_byte, length, headers, body = next(doc_iters) self.assertEqual(first_byte, 1) self.assertEqual(last_byte, 8) self.assertEqual(length, 10) header_dict = HeaderKeyDict(headers) self.assertEqual(header_dict.get('Content-Type'), 'application/lunch') self.assertEqual(body.read(), 'andwiche') self.assertRaises(StopIteration, next, doc_iters)
class FakeConn(object): def __init__(self, status, headers=None, body='', **kwargs): self.status = status try: self.reason = RESPONSE_REASONS[self.status][0] except Exception: self.reason = 'Fake' self.body = body self.resp_headers = HeaderKeyDict() if headers: self.resp_headers.update(headers) self.with_exc = False self.etag = None def _update_raw_call_args(self, *args, **kwargs): capture_attrs = ('host', 'port', 'method', 'path', 'req_headers', 'query_string') for attr, value in zip(capture_attrs, args[:len(capture_attrs)]): setattr(self, attr, value) return self def getresponse(self): if self.etag: self.resp_headers['etag'] = str(self.etag.hexdigest()) if self.with_exc: raise Exception('test') return self def getheader(self, header, default=None): return self.resp_headers.get(header, default) def getheaders(self): return self.resp_headers.items() def read(self, amt=None): if amt is None: return self.body elif isinstance(self.body, six.StringIO): return self.body.read(amt) else: return Exception('Not a StringIO entry') def send(self, data): if not self.etag: self.etag = md5() self.etag.update(data)
class FakeConn(object): def __init__(self, status, headers=None, body='', **kwargs): self.status = status try: self.reason = RESPONSE_REASONS[self.status][0] except Exception: self.reason = 'Fake' self.body = body self.resp_headers = HeaderKeyDict() if headers: self.resp_headers.update(headers) self.with_exc = False self.etag = None def _update_raw_call_args(self, *args, **kwargs): capture_attrs = ('host', 'port', 'method', 'path', 'req_headers', 'query_string') for attr, value in zip(capture_attrs, args[:len(capture_attrs)]): setattr(self, attr, value) return self def getresponse(self): if self.etag: self.resp_headers['etag'] = str(self.etag.hexdigest()) if self.with_exc: raise Exception('test') return self def getheader(self, header, default=None): return self.resp_headers.get(header, default) def getheaders(self): return self.resp_headers.items() def read(self, amt=None): if isinstance(self.body, six.StringIO): return self.body.read(amt) elif amt is None: return self.body else: return Exception('Not a StringIO entry') def send(self, data): if not self.etag: self.etag = md5() self.etag.update(data)
def test_extract_metadata(self): self.app.register('HEAD', '/v1/a/c?extract-archive=tar', HTTPNoContent, {}, None) self.app.register('PUT', '/v1/a/c/obj1?extract-archive=tar', HTTPCreated, {}, None) self.app.register('PUT', '/v1/a/c/obj2?extract-archive=tar', HTTPCreated, {}, None) # It's a real pain to instantiate TarInfo objects directly; they # really want to come from a file on disk or a tarball. So, we write # out some files and add pax headers to them as they get placed into # the tarball. with open(os.path.join(self.testdir, "obj1"), "w") as fh1: fh1.write("obj1 contents\n") with open(os.path.join(self.testdir, "obj2"), "w") as fh2: fh2.write("obj2 contents\n") tar_ball = StringIO() tar_file = tarfile.TarFile.open(fileobj=tar_ball, mode="w", format=tarfile.PAX_FORMAT) # With GNU tar 1.27.1 or later (possibly 1.27 as well), a file with # extended attribute user.thingy = dingy gets put into the tarfile # with pax_headers containing key/value pair # (SCHILY.xattr.user.thingy, dingy), both unicode strings (py2: type # unicode, not type str). # # With BSD tar (libarchive), you get key/value pair # (LIBARCHIVE.xattr.user.thingy, dingy), which strikes me as # gratuitous incompatibility. # # Still, we'll support uploads with both. Just heap more code on the # problem until you can forget it's under there. with open(os.path.join(self.testdir, "obj1")) as fh1: tar_info1 = tar_file.gettarinfo(fileobj=fh1, arcname="obj1") tar_info1.pax_headers[u'SCHILY.xattr.user.mime_type'] = \ u'application/food-diary' tar_info1.pax_headers[u'SCHILY.xattr.user.meta.lunch'] = \ u'sopa de albóndigas' tar_info1.pax_headers[ u'SCHILY.xattr.user.meta.afternoon-snack'] = \ u'gigantic bucket of coffee' tar_file.addfile(tar_info1, fh1) with open(os.path.join(self.testdir, "obj2")) as fh2: tar_info2 = tar_file.gettarinfo(fileobj=fh2, arcname="obj2") tar_info2.pax_headers[ u'LIBARCHIVE.xattr.user.meta.muppet'] = u'bert' tar_info2.pax_headers[ u'LIBARCHIVE.xattr.user.meta.cat'] = u'fluffy' tar_info2.pax_headers[ u'LIBARCHIVE.xattr.user.notmeta'] = u'skipped' tar_file.addfile(tar_info2, fh2) tar_ball.seek(0) req = Request.blank('/v1/a/c?extract-archive=tar') req.environ['REQUEST_METHOD'] = 'PUT' req.environ['wsgi.input'] = tar_ball req.headers['transfer-encoding'] = 'chunked' req.headers['accept'] = 'application/json;q=1.0' resp = req.get_response(self.bulk) self.assertEqual(resp.status_int, 200) # sanity check to make sure the upload worked upload_status = utils.json.loads(resp.body) self.assertEqual(upload_status['Number Files Created'], 2) put1_headers = HeaderKeyDict(self.app.calls_with_headers[1][2]) self.assertEqual( put1_headers.get('Content-Type'), 'application/food-diary') self.assertEqual( put1_headers.get('X-Object-Meta-Lunch'), 'sopa de alb\xc3\xb3ndigas') self.assertEqual( put1_headers.get('X-Object-Meta-Afternoon-Snack'), 'gigantic bucket of coffee') put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2]) self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert') self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy') self.assertEqual(put2_headers.get('Content-Type'), None) self.assertEqual(put2_headers.get('X-Object-Meta-Blah'), None)
def test_extract_metadata(self): self.app.register('HEAD', '/v1/a/c?extract-archive=tar', HTTPNoContent, {}, None) self.app.register('PUT', '/v1/a/c/obj1?extract-archive=tar', HTTPCreated, {}, None) self.app.register('PUT', '/v1/a/c/obj2?extract-archive=tar', HTTPCreated, {}, None) # It's a real pain to instantiate TarInfo objects directly; they # really want to come from a file on disk or a tarball. So, we write # out some files and add pax headers to them as they get placed into # the tarball. with open(os.path.join(self.testdir, "obj1"), "w") as fh1: fh1.write("obj1 contents\n") with open(os.path.join(self.testdir, "obj2"), "w") as fh2: fh2.write("obj2 contents\n") tar_ball = BytesIO() tar_file = tarfile.TarFile.open(fileobj=tar_ball, mode="w", format=tarfile.PAX_FORMAT) # With GNU tar 1.27.1 or later (possibly 1.27 as well), a file with # extended attribute user.thingy = dingy gets put into the tarfile # with pax_headers containing key/value pair # (SCHILY.xattr.user.thingy, dingy), both unicode strings (py2: type # unicode, not type str). # # With BSD tar (libarchive), you get key/value pair # (LIBARCHIVE.xattr.user.thingy, dingy), which strikes me as # gratuitous incompatibility. # # Still, we'll support uploads with both. Just heap more code on the # problem until you can forget it's under there. with open(os.path.join(self.testdir, "obj1")) as fh1: tar_info1 = tar_file.gettarinfo(fileobj=fh1, arcname="obj1") tar_info1.pax_headers[u'SCHILY.xattr.user.mime_type'] = \ u'application/food-diary' tar_info1.pax_headers[u'SCHILY.xattr.user.meta.lunch'] = \ u'sopa de albóndigas' tar_info1.pax_headers[ u'SCHILY.xattr.user.meta.afternoon-snack'] = \ u'gigantic bucket of coffee' tar_file.addfile(tar_info1, fh1) with open(os.path.join(self.testdir, "obj2")) as fh2: tar_info2 = tar_file.gettarinfo(fileobj=fh2, arcname="obj2") tar_info2.pax_headers[ u'LIBARCHIVE.xattr.user.meta.muppet'] = u'bert' tar_info2.pax_headers[ u'LIBARCHIVE.xattr.user.meta.cat'] = u'fluffy' tar_info2.pax_headers[ u'LIBARCHIVE.xattr.user.notmeta'] = u'skipped' tar_file.addfile(tar_info2, fh2) tar_ball.seek(0) req = Request.blank('/v1/a/c?extract-archive=tar') req.environ['REQUEST_METHOD'] = 'PUT' req.environ['wsgi.input'] = tar_ball req.headers['transfer-encoding'] = 'chunked' req.headers['accept'] = 'application/json;q=1.0' resp = req.get_response(self.bulk) self.assertEqual(resp.status_int, 200) # sanity check to make sure the upload worked upload_status = utils.json.loads(resp.body) self.assertEqual(upload_status['Number Files Created'], 2) put1_headers = HeaderKeyDict(self.app.calls_with_headers[1][2]) self.assertEqual(put1_headers.get('Content-Type'), 'application/food-diary') self.assertEqual(put1_headers.get('X-Object-Meta-Lunch'), 'sopa de alb\xc3\xb3ndigas') self.assertEqual(put1_headers.get('X-Object-Meta-Afternoon-Snack'), 'gigantic bucket of coffee') put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2]) self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert') self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy') self.assertEqual(put2_headers.get('Content-Type'), None) self.assertEqual(put2_headers.get('X-Object-Meta-Blah'), None)