def test_posts(self) -> None:
"""Test posts and json."""
@posts(
["https://app.fakejson.com/q"] * 3,
jsons=[{
"token": "FOk7RjbecxtWJHljGjCNjg",
"data": {
"colorText": "colorText",
"colorHex": "colorHex",
"colorRGB": "colorRGB",
"colorHSL": "colorHSL",
},
}] * 3,
)
def crawler(res: Res) -> Iterable[Json]:
"""Test crawler."""
return map(lambda r: r.json(), res)
self.assertEqual(
list(flatten(crawler())),
[{
"colorText": "tufts blue",
"colorRGB": "rgb(22, 75, 56)",
"colorHex": "colorHex",
"colorHSL": "hsl(233, 14%, 14%)",
}] * 3,
)
def test_gets_xpath(self) -> None:
"""Test gets and xpath."""
@gets(["python.org", "python.org"])
@xpath("//*[@class='widget-title']", first=True)
@text
def crawler(res: str) -> str:
"""Test crawler."""
return res
self.assertEqual(list(flatten(crawler())),
["Get Started", "Get Started"])
def test_parallel_html(self) -> None:
"""Test parallel."""
@urls(["python.org", "python.org"])
@parallel()
@get()
@html
def crawler(res: Res) -> str:
"""Test crawler."""
return res.find(".widget-title", first=True).text
self.assertEqual(list(flatten(crawler())),
["Get Started", "Get Started"])
def test_urls_text(self) -> None:
"""Test urls and text."""
@urls(["python.org", "python.org"])
@gets()
@css(".widget-title", first=True)
@text
def crawler(res: str) -> str:
"""Test crawler."""
return res
self.assertEqual(list(flatten(crawler())),
["Get Started", "Get Started"])
def more(reviews: List[Review]) -> Iterable[Review]:
"""More information."""
@urls((ReviewURL.format(rid=e.id)
for e in flatten(reviews, ignore=(Review, ))))
@parallel(4)
@get(headers=headers)
@json
def _(res: Json) -> Dict[str, str]:
"""_"""
return {
"useful":
str(res.get("votes", {}).get("useful_count", "")),
"useless":
str(res.get("votes", {}).get("useless_count", "")),
"content":
RP.sub("</>", NL.sub("</>",
HTML(html=res.get("html", "")).text)).replace(
"\t", "<>"),
}
return (Review(**{
**review._asdict(),
**r
}) for review, r in zip(reviews, flatten(_())))
@json
def _(res: Json) -> Dict[str, str]:
"""_"""
return {
"useful":
str(res.get("votes", {}).get("useful_count", "")),
"useless":
str(res.get("votes", {}).get("useless_count", "")),
"content":
RP.sub("</>", NL.sub("</>",
HTML(html=res.get("html", "")).text)).replace(
"\t", "<>"),
}
return (Review(**{
**review._asdict(),
**r
}) for review, r in zip(reviews, flatten(_())))
def write(reviews: Iterable[Review]) -> None:
"""Write out to name."""
with open(id_, "w") as f:
f.write("\n".join(map(lambda r: "\t".join(r), reviews)))
if __name__ == "__main__":
write(
flatten(more(list(flatten(getbase(), ignore=(Review, )))),
ignore=(Review, )))
def test_extra_flatten(self) -> None:
"""Test extra flatten."""
self.assertEqual(
list(
flatten(
[
1,
2,
3,
4,
"123",
[
1,
2,
3,
[
dict(a=12),
["12", "12", (1, 2, 3), (i for i in range(3))],
],
],
],
ignore=tuple(),
)),
[1, 2, 3, 4, "123", 1, 2, 3, "a", "12", "12", 1, 2, 3, 0, 1, 2],
)
self.assertEqual(
list(
flatten(
[
1,
2,
3,
4,
"123",
[
1,
2,
3,
[
dict(a=12),
["12", "12", (1, 2, 3), (i for i in range(3))],
],
],
],
ignore=(dict, tuple),
)),
[
1,
2,
3,
4,
"123",
1,
2,
3,
{
"a": 12
},
"12",
"12",
(1, 2, 3),
0,
1,
2,
],
)
self.assertEqual(
list(
flatten([
1,
2,
3,
4,
"123",
[
1,
2,
3,
[
dict(a=12),
["12", "12", (1, 2, 3), (i for i in range(3))],
],
],
])),
[
1,
2,
3,
4,
"123",
1,
2,
3,
dict(a=12),
"12",
"12",
1,
2,
3,
0,
1,
2,
],
)