async def test_qs_limit(): crawler = AsyncCrawler("http://127.0.0.1:65080/") explorer = Explorer(crawler, Event()) start_urls = deque(["http://127.0.0.1:65080/"]) excluded_urls = [] # We should have root url, huge form page, target and target with POST method assert len([__ async for __ in explorer.async_explore(start_urls, excluded_urls)]) == 4 await crawler.close() crawler = AsyncCrawler("http://127.0.0.1:65080/") explorer = Explorer(crawler, Event()) # Exclude huge POST form with limit of parameters explorer.qs_limit = 500 start_urls = deque(["http://127.0.0.1:65080/"]) excluded_urls = [] # We should have root url, huge form page, target and target with POST method assert len([__ async for __ in explorer.async_explore(start_urls, excluded_urls)]) == 3 await crawler.close()
async def test_explorer_filtering(): crawler = AsyncCrawler("http://127.0.0.1:65080/") explorer = Explorer(crawler, Event()) start_urls = deque(["http://127.0.0.1:65080/filters.html"]) excluded_urls = [] results = set([resource.url async for resource in explorer.async_explore(start_urls, excluded_urls)]) # We should have current URL and JS URL but without query string. # CSS URL should be excluded assert results == {"http://127.0.0.1:65080/filters.html", "http://127.0.0.1:65080/yolo.js"} await crawler.close()