def test_should_validate_properties_when_passing_url_and_text(self, dummy_data): response = BaseStaticResponse(url='http://foo.com', text=dummy_data.decode()) assert 'http://foo.com' == response.url assert_dicts({}, response.headers) assert_dicts({}, response.cookies) assert dummy_data.decode() == response.text assert dummy_data == response.content
def test_should_validate_attributes(self): fields = {attribute.name: attribute.type for attribute in attr.fields(BaseStaticResponse)} attributes = { '_httpx_response': Union[httpx._models.Response, type(None)], '_url': str, '_text': str, '_selector': parsel.Selector, } assert_dicts(fields, attributes)
def test_should_correctly_instantiate_class_without_giving_httpx_response(self, tmp_path): analyzer = RobotsAnalyzer(user_agent='Mozilla/5.0', robots_cache=tmp_path) assert 'Mozilla/5.0' == analyzer._user_agent assert tmp_path == analyzer._robots_cache assert isinstance(analyzer._http_client, httpx.Client) assert 'Mozilla/5.0' == analyzer._http_client.headers['User-Agent'] assert isinstance(analyzer._robots_parser, RobotFileParser) assert_dicts(analyzer._robots_mapping, {}) assert_dicts(analyzer._delay_mapping, {})
def test_should_validate_properties_when_passing_httpx_response(self): url = 'http://foobar.com' request = httpx.Request('GET', url) headers = {'foo': 'bar', 'set-cookie': 'name=John'} content = b'hello world' httpx_response = httpx.Response(200, request=request, headers=headers, content=content) response = BaseStaticResponse(httpx_response=httpx_response) assert url == response.url assert_dicts({**headers, 'content-length': '11'}, response.headers) assert_dicts({'name': 'John'}, response.cookies) assert content == response.content assert content.decode() == response.text
def test_should_return_non_empty_dict_when_scalpel_attributes_found(self): data = { 'name': 'paul', 'scalpel': { 'min_request_delay': 1, 'foo': 'bar', 'USER_AGENT': 'Mozilla/5.0', 'fruit': 'pineapple', '_config': 'foobar' } } expected = {'min_request_delay': 1, 'user_agent': 'Mozilla/5.0'} assert_dicts(expected, Configuration._scalpel_attributes(data))
def test_should_correctly_instantiate_class_with_httpx_response_passed_as_argument( self, trio_tmp_path): http_client = httpx.AsyncClient(headers={'User-Agent': 'python-httpx'}) analyzer = RobotsAnalyzer(user_agent='Mozilla/5.0', robots_cache=trio_tmp_path, http_client=http_client) assert 'Mozilla/5.0' == analyzer._user_agent assert trio_tmp_path == analyzer._robots_cache assert isinstance(analyzer._http_client, httpx.AsyncClient) assert 'python-httpx' == analyzer._http_client.headers['User-Agent'] assert isinstance(analyzer._robots_parser, RobotFileParser) assert_dicts(analyzer._robots_mapping, {}) assert_dicts(analyzer._delay_mapping, {})
def test_should_return_correct_dict_given_correct_input(self, given_dict, expected_dict): assert_dicts(expected_dict, Configuration._get_dict_with_lower_keys(given_dict))