def test_05_bad_soup(self): crawler = Crawler('/fail_soup/', bad_soup=False) self.assertTrue(crawler.crawl()) self.assertEqual(crawler.failed, 0) self.assertEqual(crawler.succeeded, 1) sio = StringIO() crawler = Crawler('/fail_soup/', bad_soup=True, output=sio) self.assertFalse(crawler.crawl()) self.assertEqual(crawler.failed, 1) self.assertEqual(crawler.succeeded, 1) self.assertSio(sio, "[SOUP] /fail_soup/ (bad end tag: u\"</scr' + 'ipt>\", at line 11, column 88)")
def test_localcrawler(self): crawler = Crawler( entry_point=settings.ENTRY_POINT, media=settings.CHECK_MEDIA, css=settings.CHECK_CSS, js=settings.CHECK_JS, bad_soup=settings.FAIL_ON_BAD_SOUP, ignore=self.ignore, client=self.client, ) success = crawler.crawl() self.assertTrue(success, "At least one URL failed to load")
def _test_type(self, name, ext=None): ext = ext or name crawler = Crawler('/fail_%s/' % name, **{name: False}) self.assertTrue(crawler.crawl()) self.assertEqual(crawler.failed, 0) self.assertEqual(crawler.succeeded, 1) sio = StringIO() crawler = Crawler('/fail_%s/' % name, output=sio, **{name: True}) self.assertFalse(crawler.crawl()) self.assertEqual(crawler.failed, 1) self.assertEqual(crawler.succeeded, 1) self.assertSio(sio, "[404] /media/does_not_exist.%s (URL Failed)" % ext) crawler = Crawler('/win_%s/' % name, **{name: True}) self.assertTrue(crawler.crawl()) self.assertEqual(crawler.failed, 0) self.assertEqual(crawler.succeeded, 2) crawler = Crawler('/win_%s/' % name, **{name: False}) self.assertTrue(crawler.crawl()) self.assertEqual(crawler.failed, 0) self.assertEqual(crawler.succeeded, 1)
def test_localcrawler(self): crawler = Crawler( entry_point=settings.ENTRY_POINT, img=settings.CHECK_IMG, media=settings.CHECK_MEDIA, # Deprecated: use CHECK_IMG media_dir=settings.CHECK_MEDIA_DIR, static_dir=settings.CHECK_STATIC_DIR, css=settings.CHECK_CSS, js=settings.CHECK_JS, bad_soup=settings.FAIL_ON_BAD_SOUP, ignore=self.ignore, client=self.client, ) success = crawler.crawl() self.assertTrue(success, "At least one URL failed to load")
def test_06_links(self): crawler = Crawler('/first/') self.assertTrue(crawler.crawl()) self.assertEqual(crawler.failed, 0) self.assertEqual(crawler.succeeded, 2)
def test_localcrawler(self): crawler = Crawler(settings.ENTRY_POINT, settings.CHECK_MEDIA, settings.CHECK_CSS, settings.CHECK_JS, settings.FAIL_ON_BAD_SOUP, self.client) success = crawler.crawl() self.assertTrue(success, "At least one URL failed to load")