def test_backend(backend): # Graph graph = graphs.Manager() graph.add_site_list(SITE_LIST) # Frontier settings = Settings() settings.BACKEND = backend settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False frontier = FrontierManager.from_settings(settings) print '-' * 80 print frontier.backend.name print '-' * 80 # Tester tester = FrontierTester(frontier, graph) tester.run() # Show crawling sequence for page in tester.sequence: print page.url
def test_backend(backend): # Graph graph = graphs.Manager() graph.add_site_list(SITE_LIST) # Frontier settings = Settings() settings.BACKEND = backend settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False frontier = FrontierManager.from_settings(settings) print '-'*80 print frontier.backend.name print '-'*80 # Tester tester = FrontierTester(frontier, graph) tester.run() # Show crawling sequence for page in tester.sequence: print page.url
def test_logic(backend): # Graph graph = graphs.Manager('sqlite:///data/graph.db') # Frontier settings = Settings() settings.BACKEND = backend settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False settings.TEST_MODE = True frontier = FrontierManager.from_settings(settings) # Tester tester = FrontierTester(frontier, graph) tester.run(add_all_pages=True) # Show crawling sequence print('-' * 80) print(frontier.backend.name) print('-' * 80) for page in tester.sequence: print(page.url)
def test_logic(backend): # Graph graph = graphs.Manager('sqlite:///data/graph.db') # Frontier settings = Settings() settings.BACKEND = backend settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False settings.TEST_MODE = True frontier = FrontierManager.from_settings(settings) # Tester tester = FrontierTester(frontier, graph) tester.run(add_all_pages=True) # Show crawling sequence print '-'*80 print frontier.backend.name print '-'*80 for page in tester.sequence: print page.url
import re import requests from frontera.contrib.requests.manager import RequestsFrontierManager from frontera import Settings from six.moves.urllib.parse import urljoin SETTINGS = Settings() SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO' SETTINGS.LOGGING_MANAGER_ENABLED = True SETTINGS.LOGGING_BACKEND_ENABLED = True SETTINGS.MAX_REQUESTS = 100 SETTINGS.MAX_NEXT_REQUESTS = 10 SEEDS = [ 'http://www.imdb.com', ] LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I) def extract_page_links(response): return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)] if __name__ == '__main__':
from frontera.core.models import Request as FrontierRequest from frontera.utils.converters import BaseRequestConverter from frontera.contrib.requests.converters import ResponseConverter from frontera.utils.managers import FrontierManagerWrapper from frontera.core import get_slot_key from frontera import Settings from six import iteritems from six.moves.urllib.parse import urljoin SETTINGS = Settings() SETTINGS.BACKEND = 'frontera.contrib.backends.memory.MemoryDFSOverusedBackend' SETTINGS.LOGGING_MANAGER_ENABLED = True SETTINGS.LOGGING_BACKEND_ENABLED = False SETTINGS.MAX_REQUESTS = 0 SETTINGS.MAX_NEXT_REQUESTS = 40 SEEDS = [ 'http://www.imdb.com', 'http://www.bbc.com/', 'http://www.amazon.com/' ] LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I) class GRequestsConverter(BaseRequestConverter): """Converts between frontera and grequests request objects"""
""" Frontier tester usage example """ from frontera import FrontierManager, FrontierTester, Settings, graphs if __name__ == '__main__': # Graph graph = graphs.Manager('sqlite:///data/graph.db') # Frontier settings = Settings() settings.TEST_MODE = True settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False frontier = FrontierManager.from_settings(settings) # Tester tester = FrontierTester(frontier, graph) # Run test tester.run() # Show crawling sequence for page in tester.sequence: print page.url
""" Frontier tester usage example """ from frontera import FrontierManager, FrontierTester, Settings, graphs if __name__ == '__main__': # Graph graph = graphs.Manager('sqlite:///data/graph.db') # Frontier settings = Settings() settings.TEST_MODE = True settings.LOGGING_MANAGER_ENABLED = True settings.LOGGING_BACKEND_ENABLED = True settings.LOGGING_DEBUGGING_ENABLED = False frontier = FrontierManager.from_settings(settings) # Tester tester = FrontierTester(frontier, graph) # Run test tester.run() # Show crawling sequence for page in tester.sequence: print(page.url)