def test_backend(backend): # Graph graph = graphs.Manager() graph.add_site_list(SITE_LIST) # Frontier settings = Settings() settings.BACKEND = backend settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False frontier = FrontierManager.from_settings(settings) print '-' * 80 print frontier.backend.name print '-' * 80 # Tester tester = FrontierTester(frontier, graph) tester.run() # Show crawling sequence for page in tester.sequence: print page.url
def __init__(self): def log(msg): print "Test Manager: ", msg self.logger = TestManager.Nothing() self.settings = Settings() self.logger.backend = TestManager.Nothing() for log_level in ('info' 'debug', 'warning', 'error'): setattr(self.logger.backend, log_level, log)
def get_settings(self): """ Returns backend settings """ return Settings( attributes={ 'BACKEND': self.backend_class, 'STRATEGY': 'tests.backends.BasicCrawlingStrategy' })
def test_overused(): settings = Settings(attributes={'OVERUSED_BATCH_DELAY': 2}) backend = Backend(settings) backend.frontier_start() backend.set_overused(0, ['a']) assert backend.get_overused_for_batch([0, 1]) == {0: {'a'}, 1: set()} backend.set_overused(0, ['a', 'b']) assert backend.get_overused_for_batch([0, 1]) == {0: {'a', 'b'}, 1: set()} backend.set_overused(1, ['a']) assert backend.get_overused_for_batch([0, 1]) == {0: {'b'}, 1: {'a'}} assert backend.get_overused_for_batch([0, 1]) == {0: set(), 1: {'a'}} assert backend.get_overused_for_batch([0, 1]) == {0: set(), 1: set()}
def test_logic(backend): # Graph graph = graphs.Manager('sqlite:///data/graph.db') # Frontier settings = Settings() settings.BACKEND = backend settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False settings.TEST_MODE = True frontier = FrontierManager.from_settings(settings) # Tester tester = FrontierTester(frontier, graph) tester.run(add_all_pages=True) # Show crawling sequence print('-' * 80) print(frontier.backend.name) print('-' * 80) for page in tester.sequence: print(page.url)
import re import requests from frontera.contrib.requests.manager import RequestsFrontierManager from frontera import Settings from six.moves.urllib.parse import urljoin SETTINGS = Settings() SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO' SETTINGS.LOGGING_MANAGER_ENABLED = True SETTINGS.LOGGING_BACKEND_ENABLED = True SETTINGS.MAX_REQUESTS = 100 SETTINGS.MAX_NEXT_REQUESTS = 10 SEEDS = [ 'http://www.imdb.com', ] LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I) def extract_page_links(response): return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)] if __name__ == '__main__':
def get_settings(self): """ Returns backend settings """ return Settings(attributes={'BACKEND': self.backend_class})
""" Frontier tester usage example """ from frontera import FrontierManager, FrontierTester, Settings, graphs if __name__ == '__main__': # Graph graph = graphs.Manager('sqlite:///data/graph.db') # Frontier settings = Settings() settings.TEST_MODE = True settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False frontier = FrontierManager.from_settings(settings) # Tester tester = FrontierTester(frontier, graph) # Run test tester.run() # Show crawling sequence for page in tester.sequence: print(page.url)