def test_crawler_adds_links_to_consume_queue(client_fixture):
"""Test workers.crawler adds links to the consume queue."""
killer(TEST_SITE)
crawler(TEST_SITE, TEST_URI)
assert len(crawl_q.jobs) == 0
assert len(consume_q.jobs) == 1
killer(TEST_SITE)
def test_crawler_adds_links_to_crawl_queue(client_fixture_crawl):
"""Test workers.crawler adds links to the crawl queue."""
killer(TEST_SITE)
crawler(TEST_SITE, TEST_URI_CRAWL)
assert len(crawl_q.jobs) == EXPECTED_JOBS
assert len(consume_q.jobs) == 0
killer(TEST_SITE)
def test_crawler_obtains_expiry_from_config(client_fixture):
killer(TEST_SITE)
crawler(TEST_SITE, TEST_URI)
visited_set, visited_uri_set, consume_set, crawl_set = get_site_sets(
TEST_SITE
)
assert visited_set.ttl() <= EXPECTED_TTL
assert visited_uri_set.ttl() <= EXPECTED_TTL
assert consume_set.ttl() <= EXPECTED_TTL
assert crawl_set.ttl() <= EXPECTED_TTL