Пример #1
0
def test_crawler_adds_links_to_consume_queue(client_fixture):
    """Test workers.crawler adds links to the consume queue."""
    killer(TEST_SITE)
    crawler(TEST_SITE, TEST_URI)

    assert len(crawl_q.jobs) == 0
    assert len(consume_q.jobs) == 1

    killer(TEST_SITE)
Пример #2
0
def test_crawler_adds_links_to_crawl_queue(client_fixture_crawl):
    """Test workers.crawler adds links to the crawl queue."""
    killer(TEST_SITE)
    crawler(TEST_SITE, TEST_URI_CRAWL)

    assert len(crawl_q.jobs) == EXPECTED_JOBS
    assert len(consume_q.jobs) == 0

    killer(TEST_SITE)
Пример #3
0
def test_crawler_obtains_expiry_from_config(client_fixture):
    killer(TEST_SITE)
    crawler(TEST_SITE, TEST_URI)
    visited_set, visited_uri_set, consume_set, crawl_set = get_site_sets(
        TEST_SITE
    )

    assert visited_set.ttl() <= EXPECTED_TTL
    assert visited_uri_set.ttl() <= EXPECTED_TTL
    assert consume_set.ttl() <= EXPECTED_TTL
    assert crawl_set.ttl() <= EXPECTED_TTL