def test_05_bad_soup(self):
crawler = Crawler('/fail_soup/', bad_soup=False)
self.assertTrue(crawler.crawl())
self.assertEqual(crawler.failed, 0)
self.assertEqual(crawler.succeeded, 1)
sio = StringIO()
crawler = Crawler('/fail_soup/', bad_soup=True, output=sio)
self.assertFalse(crawler.crawl())
self.assertEqual(crawler.failed, 1)
self.assertEqual(crawler.succeeded, 1)
self.assertSio(sio, "[SOUP] /fail_soup/ (bad end tag: u\"</scr' + 'ipt>\", at line 11, column 88)")
def test_localcrawler(self):
crawler = Crawler(
entry_point=settings.ENTRY_POINT,
media=settings.CHECK_MEDIA,
css=settings.CHECK_CSS,
js=settings.CHECK_JS,
bad_soup=settings.FAIL_ON_BAD_SOUP,
ignore=self.ignore,
client=self.client,
)
success = crawler.crawl()
self.assertTrue(success, "At least one URL failed to load")
def _test_type(self, name, ext=None):
ext = ext or name
crawler = Crawler('/fail_%s/' % name, **{name: False})
self.assertTrue(crawler.crawl())
self.assertEqual(crawler.failed, 0)
self.assertEqual(crawler.succeeded, 1)
sio = StringIO()
crawler = Crawler('/fail_%s/' % name, output=sio, **{name: True})
self.assertFalse(crawler.crawl())
self.assertEqual(crawler.failed, 1)
self.assertEqual(crawler.succeeded, 1)
self.assertSio(sio, "[404] /media/does_not_exist.%s (URL Failed)" % ext)
crawler = Crawler('/win_%s/' % name, **{name: True})
self.assertTrue(crawler.crawl())
self.assertEqual(crawler.failed, 0)
self.assertEqual(crawler.succeeded, 2)
crawler = Crawler('/win_%s/' % name, **{name: False})
self.assertTrue(crawler.crawl())
self.assertEqual(crawler.failed, 0)
self.assertEqual(crawler.succeeded, 1)
def test_localcrawler(self):
crawler = Crawler(
entry_point=settings.ENTRY_POINT,
img=settings.CHECK_IMG,
media=settings.CHECK_MEDIA, # Deprecated: use CHECK_IMG
media_dir=settings.CHECK_MEDIA_DIR,
static_dir=settings.CHECK_STATIC_DIR,
css=settings.CHECK_CSS,
js=settings.CHECK_JS,
bad_soup=settings.FAIL_ON_BAD_SOUP,
ignore=self.ignore,
client=self.client,
)
success = crawler.crawl()
self.assertTrue(success, "At least one URL failed to load")
def test_06_links(self):
crawler = Crawler('/first/')
self.assertTrue(crawler.crawl())
self.assertEqual(crawler.failed, 0)
self.assertEqual(crawler.succeeded, 2)
def test_localcrawler(self):
crawler = Crawler(settings.ENTRY_POINT, settings.CHECK_MEDIA,
settings.CHECK_CSS, settings.CHECK_JS,
settings.FAIL_ON_BAD_SOUP, self.client)
success = crawler.crawl()
self.assertTrue(success, "At least one URL failed to load")
