def test_value_scraper(self):
page1_html = '<html><body><p class="test">test</p><p>bla</p></body></html>'
page1 = Page(page1_html)
page2_html = '<html><body><div></div><p class="test">hallo</p></body></html>'
page2 = Page(page2_html)
vs = ValueScraper(CssRuleSelector(".test"), TextValueExtractor())
assert vs.get(page1) == "test"
assert vs.get(page2) == "hallo"
def test_make_matcher_for_samples():
page1_html = '<html><body><p class="test">test</p><p>bla</p></body></html>'
page1 = Page(page1_html)
sample1 = Sample(page1, "test")
page2_html = '<html><body><div></div><p class="test">hallo</p></body></html>'
page2 = Page(page2_html)
sample2 = Sample(page2, "hallo")
samples = [sample1, sample2]
assert make_matcher_for_samples(samples).selector.css_rule in ["p.test", ".test"]
def test_generate_selector_for_nodes():
page1_html = '<html><body><p class="test">test</p><p>bla</p></body></html>'
page1 = Page(page1_html)
sample1 = Sample(page1, "test")
page2_html = '<html><body><div></div><p class="test">hallo</p></body></html>'
page2 = Page(page2_html)
sample2 = Sample(page2, "hallo")
samples = [sample1, sample2]
nodes = [s.get_matches()[0].get_root() for s in samples]
gen = generate_selector_for_nodes(nodes, None)
# todo .test is also possible
assert ["p.test"] == [sel.css_rule for sel in gen]
def test_get_matches_list_of_dicts(self):
page_html = (
"<html><body>"
'<div><p class="title">Herr</p><p class="name">Lorey</p></div> '
'<div><p class="title">Frau</p><p class="name">Müller</p></div> '
"</body></html>")
page = Page(page_html)
sample = Sample(
page,
[{
"title": "Herr",
"name": "Lorey"
}, {
"title": "Frau",
"name": "Müller"
}],
)
matches = sample.get_matches()
# check that matches returns one possible list match
assert len(matches) == 1
# check that matched list item is dict
match = matches[0]
assert isinstance(match, ListMatch)
assert len(match.matches) == 2
assert all(isinstance(m, DictMatch) for m in match.matches)
print(match.get_root())
print(match.get_span())
def test_get_matches_list_basic(self):
item_htmls = map(lambda i: f"<li>{i}</li>", [1, 2, 2, 4])
body_html = f"<ul>{''.join(item_htmls)}</ul>"
page_html = f"<html><body>{body_html}</body></html>"
page = Page(page_html)
sample = Sample(page, ["1", "2", "2", "4"])
matches = sample.get_matches()
# todo check duplicate generation
# assert len(matches) == 2
assert all(isinstance(m, ListMatch) for m in matches)
def test_scrape_matches(self):
item = {"h": "no 1", "t": "the first one"}
elem_temp = "<div><h1>%(h)s</h1><p>%(t)s</p></div>"
elem = elem_temp % item
html = f"<html><body>{elem}</body></html>"
page = Page(html)
text_extractor = TextValueExtractor()
ds = DictScraper(
scraper_per_key={
"h": ValueScraper(CssRuleSelector("h1"), text_extractor),
"t": ValueScraper(CssRuleSelector("p"), text_extractor),
})
assert ds.get(page) == item
def stackoverflow_training_set():
with open("tests/static/so.html") as file:
page = Page(file.read())
item = [
{
"user": "******",
"upvotes": "20",
"when": "2011-06-16 19:45:11Z",
},
{
"user": "******",
"upvotes": "16",
"when": "2017-09-06 15:27:16Z",
},
{
"user": "******",
"upvotes": "0",
"when": "2021-01-06 10:50:04Z",
},
]
return make_training_set([page], [item])
def stackoverflow_samples():
with open("tests/static/so.html") as file:
page = Page(file.read())
item = [
{
"user": "******",
"upvotes": "20",
"when": "2011-06-16 19:45:11Z",
},
{
"user": "******",
"upvotes": "16",
"when": "2017-09-06 15:27:16Z",
},
{
"user": "******",
"upvotes": "0",
"when": "2021-01-06 10:50:04Z",
},
]
samples = [Sample(page, item)]
return samples
def main():
with open("tests/static/so.html") as file:
page = Page(file.read())
item = [
{
"user": "******",
"upvotes": "20",
"when": "2011-06-16 19:45:11Z",
},
{
"user": "******",
"upvotes": "16",
"when": "2017-09-06 15:27:16Z",
},
{
"user": "******",
"upvotes": "0",
"when": "2021-01-06 10:50:04Z",
},
]
ts = make_training_set([page], [item])
scraper = train_scraper(ts.item)
print(scraper)
def select_one(self, page: Page):
return page.select(self.css_rule)[0]
def test_generate_css_selectors_for_samples():
with open("tests/static/so.html") as file:
page = Page(file.read())
samples = [Sample(page, ["20", "16", "0"])]
selector_first = next(generate_matchers_for_samples(samples=samples))
assert selector_first.endswith(".js-vote-count")
def test_make_training_set(self):
pages = [Page(""), Page("")]
items = [{"a": "1", "b": "2"}, {"a": "3", "b": "4"}]
make_training_set(pages, items)
def test_get_matches_dict_basic(self):
page_html = "<html><body><h1>test</h1><p>2010</p><div class='footer'>2010</div></body></html>"
s = Sample(Page(page_html), {"h": "test", "year": "2010"})
matches = s.get_matches()
assert len(matches) == 2
assert all(isinstance(m, DictMatch) for m in matches)
def test_make_training_set_error(self):
pages = [Page(""), Page("")]
items = [{"a": "1", "b": "2"}, {"a": "3", "b": []}]
with pytest.raises(ItemStructureException):
make_training_set(pages, items)
def test_find_all(self):
with open("tests/static/so.html") as file:
page = Page(file.read())
nodes = page.find_all("/users/624900/jterrace")
assert nodes
def test_something(self):
with open("tests/static/so.html") as file:
page = Page(file.read())
nodes = page.select(".answer .js-vote-count")
assert [n.text for n in nodes] == ["20", "16", "0"]