def test_header_rewrite_gzipped(self):
headers = [('Content-Length', '199999'),
('Content-Type', 'text/javascript'),
('Content-Encoding', 'gzip'),
('Transfer-Encoding', 'chunked')]
rwinfo = self.do_rewrite('200 OK', headers)
# Content-Encoding, Content-Length not yet rewritten
res = """\
HTTP/1.0 200 OK\r\n\
Content-Length: 199999\r\n\
Content-Type: text/javascript\r\n\
Content-Encoding: gzip\r\n\
X-Archive-Orig-Transfer-Encoding: chunked\r\n\
"""
http_headers = DefaultHeaderRewriter(rwinfo)()
assert str(http_headers) == res
assert rwinfo.text_type == 'js'
assert rwinfo.charset == None
# access stream
stream = rwinfo.content_stream
# Content-Encoding, Content-Length rewritten now
res = """\
HTTP/1.0 200 OK\r\n\
X-Archive-Orig-Content-Length: 199999\r\n\
Content-Type: text/javascript\r\n\
X-Archive-Orig-Content-Encoding: gzip\r\n\
X-Archive-Orig-Transfer-Encoding: chunked\r\n\
"""
http_headers = DefaultHeaderRewriter(rwinfo)()
assert str(http_headers) == res
def test_header_rewrite_binary(self):
headers = [('Content-Length', '200000'), ('Content-Type', 'image/png'),
('Set-Cookie', 'foo=bar; Path=/; abc=123; Path=/path.html'),
('Content-Encoding', 'gzip'),
('Transfer-Encoding', 'chunked'), ('X-Custom', 'test'),
('Status', '200')]
rwinfo = self.do_rewrite('200 OK', headers)
http_headers = DefaultHeaderRewriter(rwinfo)()
assert (('Content-Length', '200000')) in http_headers.headers
assert (('Content-Type', 'image/png')) in http_headers.headers
assert (('Set-Cookie',
'foo=bar; Path=/warc/20171226/http://example.com/')
in http_headers.headers)
assert (('Set-Cookie',
'abc=123; Path=/warc/20171226/http://example.com/path.html')
in http_headers.headers)
assert (('Content-Encoding', 'gzip') in http_headers.headers)
assert (('X-Archive-Orig-Transfer-Encoding', 'chunked')
in http_headers.headers)
assert (('X-Custom', 'test') in http_headers.headers)
assert (('X-Archive-Orig-Status', '200') in http_headers.headers)
assert (len(http_headers.headers) == 8)
assert rwinfo.text_type == None
assert rwinfo.charset == None
def test_header_rewrite_redirect(self):
headers = [('Connection', 'close'),
('Location', 'http://example.com/other.html')]
res = """\
HTTP/1.0 302 Redirect\r\n\
X-Archive-Orig-Connection: close\r\n\
Location: /warc/20171226/http://example.com/other.html\r\n\
"""
rwinfo = self.do_rewrite('302 Redirect', headers)
http_headers = DefaultHeaderRewriter(rwinfo)()
assert str(http_headers) == res
assert rwinfo.text_type == None
assert rwinfo.charset == None
def test_header_rewrite_200_response(self):
headers = [('Date', 'Fri, 03 Jan 2014 03:03:21 GMT'),
('Content-Length', '5'),
('Content-Type', 'text/html;charset=UTF-8')]
res = """\
HTTP/1.0 200 OK\r\n\
X-Archive-Orig-Date: Fri, 03 Jan 2014 03:03:21 GMT\r\n\
X-Archive-Orig-Content-Length: 5\r\n\
Content-Type: text/html;charset=UTF-8\r\n\
"""
rwinfo = self.do_rewrite('200 OK', headers)
http_headers = DefaultHeaderRewriter(rwinfo)()
assert str(http_headers) == res
assert rwinfo.text_type == None
assert rwinfo.charset == None