Python _load_addons Beispiele

Beispiel #1

def test_load_addons_hworker_import_replace():
    addons = [TEST_ADDON]
    settings = {'SPIDER_MIDDLEWARES': {}}
    o = EntrypointSettings()
    _load_addons(addons, settings, o)
    assert len(o.attributes) == 1
    assert o['SPIDER_MIDDLEWARES'] == {'scrapy.utils.misc.arg_to_iter': 10}

Beispiel #2

Datei: test_settings.py Projekt: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_hworker_import_replace():
    addons = [TEST_ADDON]
    settings = {'SPIDER_MIDDLEWARES': {}}
    o = EntrypointSettings()
    _load_addons(addons, settings, o)
    assert len(o.attributes) == 1
    assert o['SPIDER_MIDDLEWARES'] == {'scrapy.utils.misc.arg_to_iter': 10}

Beispiel #3

def test_load_addons_basic_usage():
    addons = [TEST_ADDON]
    settings = EntrypointSettings()
    settings.setdict({'SPIDER_MIDDLEWARES': {}})
    o = EntrypointSettings()
    _load_addons(addons, settings, o)
    assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
    assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}

Beispiel #4

Datei: test_settings.py Projekt: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_basic_usage():
    addons = [TEST_ADDON]
    settings = EntrypointSettings()
    settings.setdict({'SPIDER_MIDDLEWARES': {}})
    o = EntrypointSettings()
    _load_addons(addons, settings, o)
    assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
    assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}

Beispiel #5

def test_load_addons_hworker_fail_on_import():
    addons = [TEST_ADDON.copy()]
    addons[0]['path'] = 'hworker.some.module'
    settings = EntrypointSettings()
    settings.setdict({'SPIDER_MIDDLEWARES': {}})
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'fail'})
    with pytest.raises(ImportError):
        _load_addons(addons, settings, o)

Beispiel #6

Datei: test_settings.py Projekt: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_hworker_fail_on_import():
    addons = [TEST_ADDON.copy()]
    addons[0]['path'] = 'hworker.some.module'
    settings = EntrypointSettings()
    settings.setdict({'SPIDER_MIDDLEWARES': {}})
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'fail'})
    with pytest.raises(ImportError):
        _load_addons(addons, settings, o)

Beispiel #7

def test_load_addons_hworker_warning_on_import():
    addons = [TEST_ADDON.copy()]
    addons[0]['path'] = 'hworker.some.module'
    settings = {'SPIDER_MIDDLEWARES': {}}
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'warn'})
    _load_addons(addons, settings, o)
    assert len(o.attributes) == 1
    assert o['ON_MISSING_ADDONS'] == 'warn'
    assert settings == {'SPIDER_MIDDLEWARES': {}}

Beispiel #8

Datei: test_settings.py Projekt: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_hworker_warning_on_import():
    addons = [TEST_ADDON.copy()]
    addons[0]['path'] = 'hworker.some.module'
    settings = {'SPIDER_MIDDLEWARES': {}}
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'warn'})
    _load_addons(addons, settings, o)
    assert len(o.attributes) == 1
    assert o['ON_MISSING_ADDONS'] == 'warn'
    assert settings == {'SPIDER_MIDDLEWARES': {}}

Beispiel #9

Datei: test_settings.py Projekt: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_basic_with_defaults():
    addons = [TEST_ADDON.copy()]
    addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'}
    settings = {'SPIDER_MIDDLEWARES_BASE': {
        'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
        'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500}}
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'warn'})
    _load_addons(addons, settings, o)
    assert settings == {'SPIDER_MIDDLEWARES_BASE': {
        TEST_ADDON['path']: 10,
        'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
        'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
    }}
    assert len(o.attributes) == 3
    assert o['TEST_SETTING_A'] == 'TEST'
    assert o['ON_MISSING_ADDONS'] == 'warn'
    assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3

Beispiel #10

def test_load_addons_basic_with_defaults():
    addons = [TEST_ADDON.copy()]
    addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'}
    settings = {
        'SPIDER_MIDDLEWARES_BASE': {
            'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
            'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
        }
    }
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'warn'})
    _load_addons(addons, settings, o)
    assert settings == {
        'SPIDER_MIDDLEWARES_BASE': {
            TEST_ADDON['path']: 10,
            'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
            'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
        }
    }
    assert len(o.attributes) == 3
    assert o['TEST_SETTING_A'] == 'TEST'
    assert o['ON_MISSING_ADDONS'] == 'warn'
    assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3

Beispiel #11

def test_load_addons_void():
    addons = []
    settings, o = EntrypointSettings(), EntrypointSettings()
    _load_addons(addons, settings, o)
    assert addons == []
    assert settings.attributes == o.attributes == {}

Beispiel #12

Datei: test_settings.py Projekt: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_void():
    addons = []
    settings, o = EntrypointSettings(), EntrypointSettings()
    _load_addons(addons, settings, o)
    assert addons == []
    assert settings.attributes == o.attributes == {}