def test_load_addons_hworker_import_replace(): addons = [TEST_ADDON] settings = {'SPIDER_MIDDLEWARES': {}} o = EntrypointSettings() _load_addons(addons, settings, o) assert len(o.attributes) == 1 assert o['SPIDER_MIDDLEWARES'] == {'scrapy.utils.misc.arg_to_iter': 10}
def test_load_addons_basic_usage(): addons = [TEST_ADDON] settings = EntrypointSettings() settings.setdict({'SPIDER_MIDDLEWARES': {}}) o = EntrypointSettings() _load_addons(addons, settings, o) assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10} assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
def test_load_addons_hworker_fail_on_import(): addons = [TEST_ADDON.copy()] addons[0]['path'] = 'hworker.some.module' settings = EntrypointSettings() settings.setdict({'SPIDER_MIDDLEWARES': {}}) o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'fail'}) with pytest.raises(ImportError): _load_addons(addons, settings, o)
def test_load_addons_hworker_warning_on_import(): addons = [TEST_ADDON.copy()] addons[0]['path'] = 'hworker.some.module' settings = {'SPIDER_MIDDLEWARES': {}} o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'warn'}) _load_addons(addons, settings, o) assert len(o.attributes) == 1 assert o['ON_MISSING_ADDONS'] == 'warn' assert settings == {'SPIDER_MIDDLEWARES': {}}
def test_load_addons_basic_with_defaults(): addons = [TEST_ADDON.copy()] addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'} settings = {'SPIDER_MIDDLEWARES_BASE': { 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500}} o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'warn'}) _load_addons(addons, settings, o) assert settings == {'SPIDER_MIDDLEWARES_BASE': { TEST_ADDON['path']: 10, 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500 }} assert len(o.attributes) == 3 assert o['TEST_SETTING_A'] == 'TEST' assert o['ON_MISSING_ADDONS'] == 'warn' assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3
def test_load_addons_basic_with_defaults(): addons = [TEST_ADDON.copy()] addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'} settings = { 'SPIDER_MIDDLEWARES_BASE': { 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500 } } o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'warn'}) _load_addons(addons, settings, o) assert settings == { 'SPIDER_MIDDLEWARES_BASE': { TEST_ADDON['path']: 10, 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500 } } assert len(o.attributes) == 3 assert o['TEST_SETTING_A'] == 'TEST' assert o['ON_MISSING_ADDONS'] == 'warn' assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3
def test_load_addons_void(): addons = [] settings, o = EntrypointSettings(), EntrypointSettings() _load_addons(addons, settings, o) assert addons == [] assert settings.attributes == o.attributes == {}