def test_seq_pipeline(xs: List[int]):
ys = seq.of(xs).pipe(
seq.map(lambda x: x * 10),
seq.filter(lambda x: x > 100),
seq.fold(lambda s, x: s + x, 0),
)
assert ys == functools.reduce(lambda s, x: s + x, filter(lambda x: x > 100, map(lambda x: x * 10, xs)), 0)
def test_seq_pipeline(xs: List[int]):
mapper: Callable[[int], int] = lambda x: x * 10
predicate: Callable[[int], bool] = lambda x: x > 100
folder: Callable[[int, int], int] = lambda s, x: s + x
ys = seq.of_iterable(xs).pipe(
seq.map(mapper),
seq.filter(predicate),
seq.fold(folder, 0),
)
assert ys == functools.reduce(
lambda s, x: s + x, filter(lambda x: x > 100,
map(lambda x: x * 10, xs)), 0)
def test_seq_pipe_map(xs: List[int]):
mapper: Callable[[int], int] = lambda x: x + 1
ys = pipe(xs, seq.map(mapper))
assert isinstance(ys, Iterable)
assert [y for y in ys] == [x + 1 for x in xs]
from time import time as timer
from urllib.parse import urljoin, urlparse
from multiprocessing.pool import ThreadPool
from autoscraper import AutoScraper
from expression.core import pipe
from expression.collections import Seq, seq
standard_ebooks_url = "https://standardebooks.org/ebooks"
# Navigation Scraper
navigation_scraper = AutoScraper()
scraped_pages_urls = navigation_scraper.build(
standard_ebooks_url, ["/ebooks/?page=2", "/ebooks/?page=3"])
pages_urls = Seq(scraped_pages_urls).pipe(
seq.map(lambda page: urljoin(standard_ebooks_url, page)), )
# Page Scraper
page_scraper = AutoScraper()
books_urls = page_scraper.build(standard_ebooks_url, [
"/ebooks/ford-madox-ford/some-do-not",
"/ebooks/booth-tarkington/the-turmoil",
"/ebooks/anatole-france/penguin-island/a-w-evans",
"/ebooks/edgar-allan-poe/the-narrative-of-arthur-gordon-pym-of-nantucket"
],
update=True)
for page in pages_urls:
print(page)
urls = page_scraper.get_result_similar(page)
books_urls = list(set(books_urls + urls))
books_urls = Seq(books_urls).pipe(