Exemplos de Settings em Python

Exemplo n.º 1

Arquivo: 10_custom_backends.py Projeto: 01-/frontera

def test_backend(backend):

    # Graph
    graph = graphs.Manager()
    graph.add_site_list(SITE_LIST)

    # Frontier
    settings = Settings()
    settings.BACKEND = backend
    settings.LOGGING_MANAGER_ENABLED = True
    settings.LOGGING_BACKEND_ENABLED = True
    settings.LOGGING_DEBUGGING_ENABLED = False
    frontier = FrontierManager.from_settings(settings)

    print '-'*80
    print frontier.backend.name
    print '-'*80

    # Tester
    tester = FrontierTester(frontier, graph)
    tester.run()

    # Show crawling sequence
    for page in tester.sequence:
        print page.url

Exemplo n.º 2

    def __init__(self):
        def log(msg):
            print "Test Manager: ", msg

        self.logger = TestManager.Nothing()
        self.settings = Settings()
        self.logger.backend = TestManager.Nothing()
        for log_level in ('info' 'debug', 'warning', 'error'):
            setattr(self.logger.backend, log_level, log)

Exemplo n.º 3

 def get_settings(self):
     """
     Returns backend settings
     """
     return Settings(
         attributes={
             'BACKEND': self.backend_class,
             'STRATEGY': 'tests.backends.BasicCrawlingStrategy'
         })

Exemplo n.º 4

def test_backend(backend):

    # Graph
    graph = graphs.Manager()
    graph.add_site_list(SITE_LIST)

    # Frontier
    settings = Settings()
    settings.BACKEND = backend
    settings.LOGGING_MANAGER_ENABLED = True
    settings.LOGGING_BACKEND_ENABLED = True
    settings.LOGGING_DEBUGGING_ENABLED = False
    frontier = FrontierManager.from_settings(settings)

    print '-' * 80
    print frontier.backend.name
    print '-' * 80

    # Tester
    tester = FrontierTester(frontier, graph)
    tester.run()

    # Show crawling sequence
    for page in tester.sequence:
        print page.url

Exemplo n.º 5

def test_overused():
    settings = Settings(attributes={'OVERUSED_BATCH_DELAY': 2})

    backend = Backend(settings)
    backend.frontier_start()
    backend.set_overused(0, ['a'])
    assert backend.get_overused_for_batch([0, 1]) == {0: {'a'}, 1: set()}

    backend.set_overused(0, ['a', 'b'])
    assert backend.get_overused_for_batch([0, 1]) == {0: {'a', 'b'}, 1: set()}
    backend.set_overused(1, ['a'])
    assert backend.get_overused_for_batch([0, 1]) == {0: {'b'}, 1: {'a'}}
    assert backend.get_overused_for_batch([0, 1]) == {0: set(), 1: {'a'}}
    assert backend.get_overused_for_batch([0, 1]) == {0: set(), 1: set()}

Exemplo n.º 6

def test_logic(backend):
    # Graph
    graph = graphs.Manager('sqlite:///data/graph.db')

    # Frontier
    settings = Settings()
    settings.BACKEND = backend
    settings.LOGGING_MANAGER_ENABLED = True
    settings.LOGGING_BACKEND_ENABLED = True
    settings.LOGGING_DEBUGGING_ENABLED = False
    settings.TEST_MODE = True
    frontier = FrontierManager.from_settings(settings)

    # Tester
    tester = FrontierTester(frontier, graph)
    tester.run(add_all_pages=True)

    # Show crawling sequence
    print('-' * 80)
    print(frontier.backend.name)
    print('-' * 80)
    for page in tester.sequence:
        print(page.url)

Exemplo n.º 7

Arquivo: 09_frontier_backends.py Projeto: 01-/frontera

def test_logic(backend):
    # Graph
    graph = graphs.Manager('sqlite:///data/graph.db')

    # Frontier
    settings = Settings()
    settings.BACKEND = backend
    settings.LOGGING_MANAGER_ENABLED = True
    settings.LOGGING_BACKEND_ENABLED = True
    settings.LOGGING_DEBUGGING_ENABLED = False
    settings.TEST_MODE = True
    frontier = FrontierManager.from_settings(settings)

    # Tester
    tester = FrontierTester(frontier, graph)
    tester.run(add_all_pages=True)

    # Show crawling sequence
    print '-'*80
    print frontier.backend.name
    print '-'*80
    for page in tester.sequence:
        print page.url

Exemplo n.º 8

import re

import requests

from frontera.contrib.requests.manager import RequestsFrontierManager
from frontera import Settings

from six.moves.urllib.parse import urljoin


SETTINGS = Settings()
SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'
SETTINGS.LOGGING_MANAGER_ENABLED = True
SETTINGS.LOGGING_BACKEND_ENABLED = True
SETTINGS.MAX_REQUESTS = 100
SETTINGS.MAX_NEXT_REQUESTS = 10

SEEDS = [
    'http://www.imdb.com',
]

LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I)


def extract_page_links(response):
    return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)]

if __name__ == '__main__':

Exemplo n.º 9

 def get_settings(self):
     """
     Returns backend settings
     """
     return Settings(attributes={'BACKEND': self.backend_class})

Exemplo n.º 10

Arquivo: links_follower.py Projeto: zhodj/frontera

from grequests import AsyncRequest, get as grequests_get, map as grequests_map

from frontera.core.models import Request as FrontierRequest
from frontera.utils.converters import BaseRequestConverter
from frontera.contrib.requests.converters import ResponseConverter

from frontera.utils.managers import FrontierManagerWrapper
from frontera.core import get_slot_key
from frontera import Settings

from six import iteritems
from six.moves.urllib.parse import urljoin


SETTINGS = Settings()
SETTINGS.BACKEND = 'frontera.contrib.backends.memory.MemoryDFSOverusedBackend'
SETTINGS.LOGGING_MANAGER_ENABLED = True
SETTINGS.LOGGING_BACKEND_ENABLED = False
SETTINGS.MAX_REQUESTS = 0
SETTINGS.MAX_NEXT_REQUESTS = 40

SEEDS = [
    'http://www.imdb.com',
    'http://www.bbc.com/',
    'http://www.amazon.com/'
]

LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I)

Exemplo n.º 11

Arquivo: test_frontier.py Projeto: 01-/frontera

"""
Frontier tester using recording data
"""
from frontera import FrontierManager, FrontierTester, Settings, graphs

SETTINGS = Settings()
SETTINGS.BACKEND = 'frontera.contrib.backends.memory_heapq.FIFO'
SETTINGS.LOGGING_MANAGER_ENABLED = True
SETTINGS.LOGGING_BACKEND_ENABLED = True
SETTINGS.LOGGING_DEBUGGING_ENABLED = False


if __name__ == '__main__':
    # Graph
    graph = graphs.Manager('sqlite:///recordings/scrapinghub.com.db')

    # Frontier
    frontier = FrontierManager.from_settings(SETTINGS)

    # Tester
    tester = FrontierTester(frontier, graph)

    # Run test
    tester.run()

    # Show frontier pages
    print '-'*80
    print ' Frontier pages'
    print '-'*80
    for page in frontier.backend.pages.values():
        print page.url, page.depth, page.state

Exemplo n.º 12

Arquivo: links_follower.py Projeto: 01-/frontera

import re

import requests

from urlparse import urljoin

from frontera.contrib.requests.manager import RequestsFrontierManager
from frontera import Settings

SETTINGS = Settings()
SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'
SETTINGS.LOGGING_MANAGER_ENABLED = True
SETTINGS.LOGGING_BACKEND_ENABLED = True
SETTINGS.MAX_REQUESTS = 100
SETTINGS.MAX_NEXT_REQUESTS = 10

SEEDS = [
    'http://www.imdb.com',
]

LINK_RE = re.compile(r'href="(.*?)"')


def extract_page_links(response):
    return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)]

if __name__ == '__main__':

    frontier = RequestsFrontierManager(SETTINGS)
    frontier.add_seeds([requests.Request(url=url) for url in SEEDS])
    while True:

Exemplo n.º 13

"""
Frontier tester using recording data
"""
from frontera import FrontierManager, FrontierTester, Settings, graphs

SETTINGS = Settings()
SETTINGS.BACKEND = 'frontera.contrib.backends.memory_heapq.FIFO'
SETTINGS.LOGGING_MANAGER_ENABLED = True
SETTINGS.LOGGING_BACKEND_ENABLED = True
SETTINGS.LOGGING_DEBUGGING_ENABLED = False

if __name__ == '__main__':
    # Graph
    graph = graphs.Manager('sqlite:///recordings/scrapinghub.com.db')

    # Frontier
    frontier = FrontierManager.from_settings(SETTINGS)

    # Tester
    tester = FrontierTester(frontier, graph)

    # Run test
    tester.run()

    # Show frontier pages
    print('-' * 80)
    print(' Frontier pages')
    print('-' * 80)
    for page in list(frontier.backend.pages.values()):
        print(page.url, page.depth, page.state)

Exemplo n.º 14

Arquivo: 07_frontier_from_settings.py Projeto: 01-/frontera

"""
Frontier initialization from settings
"""
from frontera import FrontierManager, Settings, graphs, Request, Response

SETTINGS = Settings()
SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'
SETTINGS.LOGGING_MANAGER_ENABLED = True
SETTINGS.LOGGING_BACKEND_ENABLED = True
SETTINGS.LOGGING_DEBUGGING_ENABLED = True
SETTINGS.TEST_MODE = True

if __name__ == '__main__':
    # Create graph
    graph = graphs.Manager('sqlite:///data/graph.db')

    # Create frontier from settings
    frontier = FrontierManager.from_settings(SETTINGS)

    # Add seeds
    frontier.add_seeds([Request(seed.url) for seed in graph.seeds])

    # Get next requests
    next_requests = frontier.get_next_requests()

    # Crawl pages
    for request in next_requests:

        # Fake page crawling
        crawled_page = graph.get_page(request.url)

Exemplo n.º 15

Arquivo: 08_frontier_tester.py Projeto: 01-/frontera

"""
Frontier tester usage example
"""
from frontera import FrontierManager, FrontierTester, Settings, graphs

if __name__ == '__main__':
    # Graph
    graph = graphs.Manager('sqlite:///data/graph.db')

    # Frontier
    settings = Settings()
    settings.TEST_MODE = True
    settings.LOGGING_MANAGER_ENABLED = True
    settings.LOGGING_BACKEND_ENABLED = True
    settings.LOGGING_DEBUGGING_ENABLED = False
    frontier = FrontierManager.from_settings(settings)

    # Tester
    tester = FrontierTester(frontier, graph)

    # Run test
    tester.run()

    # Show crawling sequence
    for page in tester.sequence:
        print page.url

Exemplo n.º 16

"""
Frontier tester usage example
"""
from frontera import FrontierManager, FrontierTester, Settings, graphs

if __name__ == '__main__':
    # Graph
    graph = graphs.Manager('sqlite:///data/graph.db')

    # Frontier
    settings = Settings()
    settings.TEST_MODE = True
    settings.LOGGING_MANAGER_ENABLED = True
    settings.LOGGING_BACKEND_ENABLED = True
    settings.LOGGING_DEBUGGING_ENABLED = False
    frontier = FrontierManager.from_settings(settings)

    # Tester
    tester = FrontierTester(frontier, graph)

    # Run test
    tester.run()

    # Show crawling sequence
    for page in tester.sequence:
        print(page.url)