def test_record_skip_http_only_cookies_header(self):
warc_path = to_path(self.root_dir + '/warcs/cookieskip_httponly/')
header_filter = ExcludeHttpOnlyCookieHeaders()
recorder_app = RecorderApp(self.upstream_url,
PerRecordWARCWriter(
warc_path, header_filter=header_filter),
accept_colls='live')
resp = self._test_warc_write(recorder_app, 'www.google.com', '/')
assert b'HTTP/1.1 302' in resp.body
buff = BytesIO(resp.body)
record = ArcWarcRecordLoader().parse_record_stream(buff)
non_http_only, http_only = self._get_http_only_cookies(record)
# both httponly and other cookies
assert http_only != None
assert non_http_only != None
stored_req, stored_resp = self._load_resp_req(warc_path)
non_http_only, http_only = self._get_http_only_cookies(stored_resp)
# no httponly cookies
assert http_only == None
assert non_http_only != None
assert ('X-Other', 'foo') in stored_req.http_headers.headers
assert ('Cookie', 'boo=far') not in stored_req.http_headers.headers
self._test_all_warcs('/warcs/cookieskip_httponly/', 1)
def init_recorder(self):
self.dedup_index = self.init_indexer()
writer = SkipCheckingMultiFileWARCWriter(
dir_template=self.warc_path_templ,
filename_template=self.warc_name_templ,
dedup_index=self.dedup_index,
redis=self.redis,
skip_key_templ=self.skip_key_templ,
key_template=self.info_keys['rec'],
header_filter=ExcludeHttpOnlyCookieHeaders())
self.writer = writer
recorder_app = RecorderApp(self.upstream_url,
writer,
accept_colls='(live|mount:)',
create_buff_func=self.create_buffer)
self.recorder = recorder_app
def init_recorder(self):
self.dedup_index = self.init_indexer()
writer = SkipCheckingMultiFileWARCWriter(dir_template=self.warc_path_templ,
dedup_index=self.dedup_index,
redis=self.redis,
key_template=self.info_keys['rec'],
header_filter=ExcludeHttpOnlyCookieHeaders(),
config=self.config)
self.writer = writer
skip_filters = [SkipRangeRequestFilter(),
ExtractPatchingFilter()]
recorder_app = RecorderApp(self.upstream_url,
writer,
skip_filters=skip_filters,
#accept_colls=self.accept_colls,
create_buff_func=writer.create_write_buffer)
self.recorder = recorder_app