def test_process_response_stores_response_body_to_file_if_request_asks_for_it(
write_to_file_mock):
downloader = HtmlStorageMiddleware(Settings())
request_mock = make_request_mock(save_html=True)
response_mock = make_response_mock(response_status=200)
downloader.process_response(request_mock, response_mock, MagicMock())
assert_that(write_to_file_mock.call_count, is_(1))
def test_process_response_stores_response_body_to_gzip_file_if_this_setting_is_on(
write_to_gzip_mock):
downloader = HtmlStorageMiddleware(Settings())
downloader.gzip_output = True
request_mock = make_request_mock(save_html=True)
response_mock = make_response_mock(response_status=200)
downloader.process_response(request_mock, response_mock, MagicMock())
assert_that(write_to_gzip_mock.call_count, is_(1))
def test_process_response_saves_response_html_to_file_resolved_by_spider(
write_to_file_mock):
downloader = HtmlStorageMiddleware(Settings())
request_mock = make_request_mock(save_html=True)
response_mock = make_response_mock(response_status=200)
spider_mock = MagicMock()
spider_mock.response_html_path.return_value = '/tmp/response.html'
downloader.process_response(request_mock, response_mock, spider_mock)
write_to_file_mock.assert_called_with('/tmp/response.html', ANY)
def make_downloader(save_html_on_codes=[]):
settings = Settings()
settings.set('HTML_STORAGE', {
'gzip_output': True,
'save_html_on_codes': save_html_on_codes
})
return HtmlStorageMiddleware(settings)
def test_from_settings_constructs_middleware_with_the_specified_settings():
settings = Settings()
settings.set('HTML_STORAGE', {'test': 'settings'})
downloader = HtmlStorageMiddleware.from_settings(settings)
assert_that(downloader.settings, is_({'test': 'settings'}))
def test_contructor_sets_default_settings_values_when_no_settings_are_specified(
setting_name, expected):
settings = Settings()
settings.set('HTML_STORAGE', {})
downloader = HtmlStorageMiddleware(settings)
assert_that(downloader.__dict__[setting_name], is_(expected))
def test_constructor_extracts_expected_settings():
settings = Settings()
save_html_on_codes = make_allowed_response_codes_list()
settings.set('HTML_STORAGE', {
'gzip_output': True,
'save_html_on_codes': save_html_on_codes
})
downloader = HtmlStorageMiddleware(settings)
assert_that(downloader, has_properties(dict(
gzip_output=True,
save_html_on_codes=save_html_on_codes
)))
def test_constructor_sets_empty_settings_when_middleware_settings_are_not_specified():
settings = Settings()
downloader = HtmlStorageMiddleware(settings)
assert_that(downloader.settings, is_({}))