def test_url_plugin(self):
conf_urls = {'this_wont_be_crawled': True}
c = Crawler('/', conf_urls=conf_urls)
c.run()
logs = open('crawler_log')
output = logs.read()
self.assertTrue(output.find('These patterns were not matched during the crawl: this_wont_be_crawled') != -1)
def test_time_plugin(self):
#This isn't testing much, but I can't know how long the time will take
c = Crawler('/')
c.run()
logs = open('crawler_log')
output = logs.read()
self.assertTrue(output.find('Time taken:') != -1)
def test_time_plugin(self):
#This isn't testing much, but I can't know how long the time will take
c = Crawler('/')
c.run()
logs = open('crawler_log')
output = logs.read()
self.assertTrue(output.find('Time taken:') != -1)
def test_memory_plugin(self):
from crawler.plugins.memory_plugin import Memory
Memory.active = True
c = Crawler('/')
c.run()
logs = open('crawler_log')
output = logs.read()
self.assertTrue(output.find('Memory consumed:') != -1)
def test_memory_plugin(self):
from crawler.plugins.memory_plugin import Memory
Memory.active = True
c = Crawler('/')
c.run()
logs = open('crawler_log')
output = logs.read()
self.assertTrue(output.find('Memory consumed:') != -1)
def test_url_plugin(self):
conf_urls = {'this_wont_be_crawled': True}
c = Crawler('/', conf_urls=conf_urls)
c.run()
logs = open('crawler_log')
output = logs.read()
self.assertTrue(
output.find(
'These patterns were not matched during the crawl: this_wont_be_crawled'
) != -1)
def test_relative_crawling(self):
c = Crawler('/1')
c.run()
self.assertEqual(c.crawled, {u'/1': True})
def test_basic_crawling(self):
c = Crawler('/')
c.run()
self.assertEqual(c.crawled, {'/': True, u'/1': True, u'/2': True})
def test_relative_crawling(self):
c = Crawler('/1')
c.run()
self.assertEqual(c.crawled, {u'/1': True})
def test_basic_crawling(self):
c = Crawler('/')
c.run()
self.assertEqual(c.crawled, {'/': True, u'/1': True, u'/2': True})
