def test_seq_pipeline(xs: List[int]): ys = seq.of(xs).pipe( seq.map(lambda x: x * 10), seq.filter(lambda x: x > 100), seq.fold(lambda s, x: s + x, 0), ) assert ys == functools.reduce(lambda s, x: s + x, filter(lambda x: x > 100, map(lambda x: x * 10, xs)), 0)
def test_seq_pipeline(xs: List[int]): mapper: Callable[[int], int] = lambda x: x * 10 predicate: Callable[[int], bool] = lambda x: x > 100 folder: Callable[[int, int], int] = lambda s, x: s + x ys = seq.of_iterable(xs).pipe( seq.map(mapper), seq.filter(predicate), seq.fold(folder, 0), ) assert ys == functools.reduce( lambda s, x: s + x, filter(lambda x: x > 100, map(lambda x: x * 10, xs)), 0)
def test_seq_pipe_map(xs: List[int]): mapper: Callable[[int], int] = lambda x: x + 1 ys = pipe(xs, seq.map(mapper)) assert isinstance(ys, Iterable) assert [y for y in ys] == [x + 1 for x in xs]
from time import time as timer from urllib.parse import urljoin, urlparse from multiprocessing.pool import ThreadPool from autoscraper import AutoScraper from expression.core import pipe from expression.collections import Seq, seq standard_ebooks_url = "https://standardebooks.org/ebooks" # Navigation Scraper navigation_scraper = AutoScraper() scraped_pages_urls = navigation_scraper.build( standard_ebooks_url, ["/ebooks/?page=2", "/ebooks/?page=3"]) pages_urls = Seq(scraped_pages_urls).pipe( seq.map(lambda page: urljoin(standard_ebooks_url, page)), ) # Page Scraper page_scraper = AutoScraper() books_urls = page_scraper.build(standard_ebooks_url, [ "/ebooks/ford-madox-ford/some-do-not", "/ebooks/booth-tarkington/the-turmoil", "/ebooks/anatole-france/penguin-island/a-w-evans", "/ebooks/edgar-allan-poe/the-narrative-of-arthur-gordon-pym-of-nantucket" ], update=True) for page in pages_urls: print(page) urls = page_scraper.get_result_similar(page) books_urls = list(set(books_urls + urls)) books_urls = Seq(books_urls).pipe(