def test_fetch_end_result(self):
parser = Parser(TEST_DOMAINS[0])
parser.add_soup_getter("header", "h1")
parser.add_soup_getter("with_property", "div", {"class": "test"})
parser.add_soup_list_getter("lis", "li")
parser.add_soup_list_getter("lis_others", "li", {"class": "other"})
parser.add_getter_function("function", test_getter_function)
parser.add_soup_getter("a_href", "a", attribute="href")
parser.add_soup_getter("a_text", "a", attribute="text")
response = mock.Mock()
response.text = TEST_HTML_P1
session = requests.session()
session.get = mock.Mock(return_value=response)
fetcher = Fetcher(parser, session=session)
result = fetcher.fetch(TEST_URL)
self.assertEqual(
result, {
"header": TEST_STRING,
"with_property": "some text",
"lis": ["a", "b", "c", "d", "e"],
"lis_others": ["d", "e"],
"function": TEST_STRING.upper(),
"a_href": TEST_DOMAINS[0],
"a_text": TEST_A_TEXT,
})
def test_parse(self):
parser = Parser()
parser.add_soup_getter("header", "h1")
parser.add_soup_getter("with_property", "div", {"class": "test"})
parser.add_soup_list_getter("lis", "li")
parser.add_soup_list_getter("lis_others", "li", {"class": "other"})
parser.add_getter_function("function", test_getter_function)
parser.add_soup_getter("a_href", "a", attribute="href")
parser.add_soup_getter("a_text", "a", attribute="text")
result = parser.parse(TEST_HTML_P1)
self.assertEqual(
result, {
"header": TEST_STRING,
"with_property": "some text",
"lis": ["a", "b", "c", "d", "e"],
"lis_others": ["d", "e"],
"function": TEST_STRING.upper(),
"a_href": TEST_DOMAINS[0],
"a_text": TEST_A_TEXT,
})
def test_getter_function_added(self):
parser = Parser()
parser.add_getter_function(TEST_PROPERTY, test_getter_function)
self.assertTrue(TEST_PROPERTY in parser.getter_functions)
self.assertTrue(
parser.getter_functions[TEST_PROPERTY] is test_getter_function)