Python EntrypointSettings Examples

Example #1

def test_update_settings_check_unicode_in_py2_key_value():
    # a dict entry is duplicated as unicode doesn't match native str value
    test = EntrypointSettings()
    test.setdict({'\xf1e\xf1e\xf1e': '\xf1e\xf1e'}, 10)
    assert test['\xf1e\xf1e\xf1e'] == '\xf1e\xf1e'
    native_key = to_native_str('\xf1e\xf1e\xf1e')
    assert test[native_key] == to_native_str('\xf1e\xf1e')

Example #2

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_check_unicode_in_py2_key_value():
    # a dict entry is duplicated as unicode doesn't match native str value
    test = EntrypointSettings()
    test.setdict({'\xf1e\xf1e\xf1e': '\xf1e\xf1e'}, 10)
    assert test['\xf1e\xf1e\xf1e'] == '\xf1e\xf1e'
    native_key = to_native_str('\xf1e\xf1e\xf1e')
    assert test[native_key] == to_native_str('\xf1e\xf1e')

Example #3

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_per_key_priorities_new_behaviour():
    from scrapy.settings import BaseSettings
    test = EntrypointSettings()
    test.set('ITEM_PIPELINES', BaseSettings())
    test['ITEM_PIPELINES'].update({'test.path1': 100})
    test['ITEM_PIPELINES'].update({'test.path2': 200})
    assert dict(test['ITEM_PIPELINES']) == {
        'test.path1': 100, 'test.path2': 200}

Example #4

def test_load_addons_basic_usage():
    addons = [TEST_ADDON]
    settings = EntrypointSettings()
    settings.setdict({'SPIDER_MIDDLEWARES': {}})
    o = EntrypointSettings()
    _load_addons(addons, settings, o)
    assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
    assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}

Example #5

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_basic_usage():
    addons = [TEST_ADDON]
    settings = EntrypointSettings()
    settings.setdict({'SPIDER_MIDDLEWARES': {}})
    o = EntrypointSettings()
    _load_addons(addons, settings, o)
    assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
    assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}

Example #6

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_update_existing_value():
    test = EntrypointSettings()
    test.setdict({'a': 'b', 'c': 'd'}, priority=10)
    test.setdict({'c': 'e', 'f': 'g'}, 10)
    assert len(test.attributes) == 3
    assert test['a'] == 'b'
    assert test['c'] == 'e'
    assert test['f'] == 'g'

Example #7

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_hworker_warning_on_import():
    addons = [TEST_ADDON.copy()]
    addons[0]['path'] = 'hworker.some.module'
    settings = {'SPIDER_MIDDLEWARES': {}}
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'warn'})
    _load_addons(addons, settings, o)
    assert len(o.attributes) == 1
    assert o['ON_MISSING_ADDONS'] == 'warn'
    assert settings == {'SPIDER_MIDDLEWARES': {}}

Example #8

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_maybe_load_autoscraping_project_ok():
    result = EntrypointSettings()
    result.setdict({'SPIDER_MANAGER_CLASS': 'test.class'})
    _maybe_load_autoscraping_project(result)
    assert result['ITEM_PIPELINES'] == {
        'slybot.dupefilter.DupeFilterPipeline': 0}
    assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip'
    assert result['SLYCLOSE_SPIDER_ENABLED']
    assert result['SLYDUPEFILTER_ENABLED']
    assert result['SPIDER_MANAGER_CLASS'] == 'test.class'

Example #9

def test_load_addons_hworker_warning_on_import():
    addons = [TEST_ADDON.copy()]
    addons[0]['path'] = 'hworker.some.module'
    settings = {'SPIDER_MIDDLEWARES': {}}
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'warn'})
    _load_addons(addons, settings, o)
    assert len(o.attributes) == 1
    assert o['ON_MISSING_ADDONS'] == 'warn'
    assert settings == {'SPIDER_MIDDLEWARES': {}}

Example #10

def test_update_settings_per_key_priorities_new_behaviour():
    from scrapy.settings import BaseSettings
    test = EntrypointSettings()
    test.set('ITEM_PIPELINES', BaseSettings())
    test['ITEM_PIPELINES'].update({'test.path1': 100})
    test['ITEM_PIPELINES'].update({'test.path2': 200})
    assert dict(test['ITEM_PIPELINES']) == {
        'test.path1': 100,
        'test.path2': 200
    }

Example #11

def test_maybe_load_autoscraping_project_ok():
    result = EntrypointSettings()
    result.setdict({'SPIDER_MANAGER_CLASS': 'test.class'})
    _maybe_load_autoscraping_project(result)
    assert result['ITEM_PIPELINES'] == {
        'slybot.dupefilter.DupeFilterPipeline': 0
    }
    assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip'
    assert result['SLYCLOSE_SPIDER_ENABLED']
    assert result['SLYDUPEFILTER_ENABLED']
    assert result['SPIDER_MANAGER_CLASS'] == 'test.class'

Example #12

def test_load_addons_hworker_import_replace():
    addons = [TEST_ADDON]
    settings = {'SPIDER_MIDDLEWARES': {}}
    o = EntrypointSettings()
    _load_addons(addons, settings, o)
    assert len(o.attributes) == 1
    assert o['SPIDER_MIDDLEWARES'] == {'scrapy.utils.misc.arg_to_iter': 10}

Example #13

def test_load_addons_hworker_fail_on_import():
    addons = [TEST_ADDON.copy()]
    addons[0]['path'] = 'hworker.some.module'
    settings = EntrypointSettings()
    settings.setdict({'SPIDER_MIDDLEWARES': {}})
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'fail'})
    with pytest.raises(ImportError):
        _load_addons(addons, settings, o)

Example #14

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_basic_with_defaults():
    addons = [TEST_ADDON.copy()]
    addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'}
    settings = {'SPIDER_MIDDLEWARES_BASE': {
        'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
        'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500}}
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'warn'})
    _load_addons(addons, settings, o)
    assert settings == {'SPIDER_MIDDLEWARES_BASE': {
        TEST_ADDON['path']: 10,
        'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
        'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
    }}
    assert len(o.attributes) == 3
    assert o['TEST_SETTING_A'] == 'TEST'
    assert o['ON_MISSING_ADDONS'] == 'warn'
    assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3

Example #15

def test_update_settings_update_existing_value():
    test = EntrypointSettings()
    test.setdict({'a': 'b', 'c': 'd'}, priority=10)
    test.setdict({'c': 'e', 'f': 'g'}, 10)
    assert len(test.attributes) == 3
    assert test['a'] == 'b'
    assert test['c'] == 'e'
    assert test['f'] == 'g'

Example #16

def test_load_addons_basic_with_defaults():
    addons = [TEST_ADDON.copy()]
    addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'}
    settings = {
        'SPIDER_MIDDLEWARES_BASE': {
            'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
            'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
        }
    }
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'warn'})
    _load_addons(addons, settings, o)
    assert settings == {
        'SPIDER_MIDDLEWARES_BASE': {
            TEST_ADDON['path']: 10,
            'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
            'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
        }
    }
    assert len(o.attributes) == 3
    assert o['TEST_SETTING_A'] == 'TEST'
    assert o['ON_MISSING_ADDONS'] == 'warn'
    assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3

Example #17

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_load_addons_hworker_fail_on_import():
    addons = [TEST_ADDON.copy()]
    addons[0]['path'] = 'hworker.some.module'
    settings = EntrypointSettings()
    settings.setdict({'SPIDER_MIDDLEWARES': {}})
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'fail'})
    with pytest.raises(ImportError):
        _load_addons(addons, settings, o)

Example #18

def test_update_settings_dont_fail_on_non_string():
    test = EntrypointSettings()
    test.setdict({'a': 3}, 10)
    assert test['a'] == 3

Example #19

def test_update_settings_base_test2():
    test = EntrypointSettings()
    test.setdict({'a': 'b', 'c': 'd'}, 10)
    assert len(test.attributes) == 2

Example #20

def test_update_settings_base_test():
    test = EntrypointSettings()
    test.setdict({'a': 'b'}, 10)
    assert test['a'] == 'b'

Example #21

def test_update_settings_void_dictionaries():
    test = EntrypointSettings()
    test.setdict({}, 10)
    assert len(test.attributes) == 0

Example #22

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_base_test():
    test = EntrypointSettings()
    test.setdict({'a': 'b'}, 10)
    assert test['a'] == 'b'

Example #23

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_get_action_on_missing_addons_warn_if_wrong_value():
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'wrong'})
    assert _get_action_on_missing_addons(o) == 'warn'

Example #24

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_base_test2():
    test = EntrypointSettings()
    test.setdict({'a': 'b', 'c': 'd'}, 10)
    assert len(test.attributes) == 2

Example #25

def test_load_addons_void():
    addons = []
    settings, o = EntrypointSettings(), EntrypointSettings()
    _load_addons(addons, settings, o)
    assert addons == []
    assert settings.attributes == o.attributes == {}

Example #26

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_get_action_on_missing_addons_base():
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'fail'})
    assert _get_action_on_missing_addons(o) == 'fail'

Example #27

def test_get_action_on_missing_addons_warn_if_wrong_value():
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'wrong'})
    assert _get_action_on_missing_addons(o) == 'warn'

Example #28

def test_get_action_on_missing_addons_base():
    o = EntrypointSettings()
    o.setdict({'ON_MISSING_ADDONS': 'fail'})
    assert _get_action_on_missing_addons(o) == 'fail'

Example #29

def test_get_action_on_missing_addons_default():
    o = EntrypointSettings()
    assert _get_action_on_missing_addons(o) == 'warn'

Example #30

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_per_key_priorities_old_behavior():
    test = EntrypointSettings()
    test.set('ITEM_PIPELINES', {'path.one': 100})
    test.set('ITEM_PIPELINES', {'path.two': 200})
    assert test['ITEM_PIPELINES'] == {'path.two': 200}

Example #31

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_void_dictionaries():
    test = EntrypointSettings()
    test.setdict({}, 10)
    assert len(test.attributes) == 0

Example #32

def test_update_settings_per_key_priorities_old_behavior():
    test = EntrypointSettings()
    test.set('ITEM_PIPELINES', {'path.one': 100})
    test.set('ITEM_PIPELINES', {'path.two': 200})
    assert test['ITEM_PIPELINES'] == {'path.two': 200}

Example #33

def test_update_settings_check_unicode_in_py3():
    test = EntrypointSettings()
    test.setdict({'\xf1e\xf1e\xf1e': 'test'}, 10)
    assert test['\xf1e\xf1e\xf1e'] == 'test'

Example #34

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_check_unicode_in_py3():
    test = EntrypointSettings()
    test.setdict({'\xf1e\xf1e\xf1e': 'test'}, 10)
    assert test['\xf1e\xf1e\xf1e'] == 'test'

Example #35

File: test_settings.py Project: scrapinghub/scrapinghub-entrypoint-scrapy

def test_update_settings_dont_fail_on_non_string():
    test = EntrypointSettings()
    test.setdict({'a': 3}, 10)
    assert test['a'] == 3