def test_update_settings_check_unicode_in_py2_key_value(): # a dict entry is duplicated as unicode doesn't match native str value test = EntrypointSettings() test.setdict({'\xf1e\xf1e\xf1e': '\xf1e\xf1e'}, 10) assert test['\xf1e\xf1e\xf1e'] == '\xf1e\xf1e' native_key = to_native_str('\xf1e\xf1e\xf1e') assert test[native_key] == to_native_str('\xf1e\xf1e')
def test_update_settings_check_unicode_in_py2_key_value(): # a dict entry is duplicated as unicode doesn't match native str value test = EntrypointSettings() test.setdict({'\xf1e\xf1e\xf1e': '\xf1e\xf1e'}, 10) assert test['\xf1e\xf1e\xf1e'] == '\xf1e\xf1e' native_key = to_native_str('\xf1e\xf1e\xf1e') assert test[native_key] == to_native_str('\xf1e\xf1e')
def test_update_settings_per_key_priorities_new_behaviour(): from scrapy.settings import BaseSettings test = EntrypointSettings() test.set('ITEM_PIPELINES', BaseSettings()) test['ITEM_PIPELINES'].update({'test.path1': 100}) test['ITEM_PIPELINES'].update({'test.path2': 200}) assert dict(test['ITEM_PIPELINES']) == { 'test.path1': 100, 'test.path2': 200}
def test_load_addons_basic_usage(): addons = [TEST_ADDON] settings = EntrypointSettings() settings.setdict({'SPIDER_MIDDLEWARES': {}}) o = EntrypointSettings() _load_addons(addons, settings, o) assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10} assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
def test_load_addons_basic_usage(): addons = [TEST_ADDON] settings = EntrypointSettings() settings.setdict({'SPIDER_MIDDLEWARES': {}}) o = EntrypointSettings() _load_addons(addons, settings, o) assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10} assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
def test_update_settings_update_existing_value(): test = EntrypointSettings() test.setdict({'a': 'b', 'c': 'd'}, priority=10) test.setdict({'c': 'e', 'f': 'g'}, 10) assert len(test.attributes) == 3 assert test['a'] == 'b' assert test['c'] == 'e' assert test['f'] == 'g'
def test_load_addons_hworker_warning_on_import(): addons = [TEST_ADDON.copy()] addons[0]['path'] = 'hworker.some.module' settings = {'SPIDER_MIDDLEWARES': {}} o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'warn'}) _load_addons(addons, settings, o) assert len(o.attributes) == 1 assert o['ON_MISSING_ADDONS'] == 'warn' assert settings == {'SPIDER_MIDDLEWARES': {}}
def test_maybe_load_autoscraping_project_ok(): result = EntrypointSettings() result.setdict({'SPIDER_MANAGER_CLASS': 'test.class'}) _maybe_load_autoscraping_project(result) assert result['ITEM_PIPELINES'] == { 'slybot.dupefilter.DupeFilterPipeline': 0} assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip' assert result['SLYCLOSE_SPIDER_ENABLED'] assert result['SLYDUPEFILTER_ENABLED'] assert result['SPIDER_MANAGER_CLASS'] == 'test.class'
def test_load_addons_hworker_warning_on_import(): addons = [TEST_ADDON.copy()] addons[0]['path'] = 'hworker.some.module' settings = {'SPIDER_MIDDLEWARES': {}} o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'warn'}) _load_addons(addons, settings, o) assert len(o.attributes) == 1 assert o['ON_MISSING_ADDONS'] == 'warn' assert settings == {'SPIDER_MIDDLEWARES': {}}
def test_update_settings_per_key_priorities_new_behaviour(): from scrapy.settings import BaseSettings test = EntrypointSettings() test.set('ITEM_PIPELINES', BaseSettings()) test['ITEM_PIPELINES'].update({'test.path1': 100}) test['ITEM_PIPELINES'].update({'test.path2': 200}) assert dict(test['ITEM_PIPELINES']) == { 'test.path1': 100, 'test.path2': 200 }
def test_maybe_load_autoscraping_project_ok(): result = EntrypointSettings() result.setdict({'SPIDER_MANAGER_CLASS': 'test.class'}) _maybe_load_autoscraping_project(result) assert result['ITEM_PIPELINES'] == { 'slybot.dupefilter.DupeFilterPipeline': 0 } assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip' assert result['SLYCLOSE_SPIDER_ENABLED'] assert result['SLYDUPEFILTER_ENABLED'] assert result['SPIDER_MANAGER_CLASS'] == 'test.class'
def test_load_addons_hworker_import_replace(): addons = [TEST_ADDON] settings = {'SPIDER_MIDDLEWARES': {}} o = EntrypointSettings() _load_addons(addons, settings, o) assert len(o.attributes) == 1 assert o['SPIDER_MIDDLEWARES'] == {'scrapy.utils.misc.arg_to_iter': 10}
def test_load_addons_hworker_fail_on_import(): addons = [TEST_ADDON.copy()] addons[0]['path'] = 'hworker.some.module' settings = EntrypointSettings() settings.setdict({'SPIDER_MIDDLEWARES': {}}) o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'fail'}) with pytest.raises(ImportError): _load_addons(addons, settings, o)
def test_load_addons_basic_with_defaults(): addons = [TEST_ADDON.copy()] addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'} settings = {'SPIDER_MIDDLEWARES_BASE': { 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500}} o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'warn'}) _load_addons(addons, settings, o) assert settings == {'SPIDER_MIDDLEWARES_BASE': { TEST_ADDON['path']: 10, 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500 }} assert len(o.attributes) == 3 assert o['TEST_SETTING_A'] == 'TEST' assert o['ON_MISSING_ADDONS'] == 'warn' assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3
def test_update_settings_update_existing_value(): test = EntrypointSettings() test.setdict({'a': 'b', 'c': 'd'}, priority=10) test.setdict({'c': 'e', 'f': 'g'}, 10) assert len(test.attributes) == 3 assert test['a'] == 'b' assert test['c'] == 'e' assert test['f'] == 'g'
def test_load_addons_basic_with_defaults(): addons = [TEST_ADDON.copy()] addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'} settings = { 'SPIDER_MIDDLEWARES_BASE': { 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500 } } o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'warn'}) _load_addons(addons, settings, o) assert settings == { 'SPIDER_MIDDLEWARES_BASE': { TEST_ADDON['path']: 10, 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500 } } assert len(o.attributes) == 3 assert o['TEST_SETTING_A'] == 'TEST' assert o['ON_MISSING_ADDONS'] == 'warn' assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3
def test_load_addons_hworker_fail_on_import(): addons = [TEST_ADDON.copy()] addons[0]['path'] = 'hworker.some.module' settings = EntrypointSettings() settings.setdict({'SPIDER_MIDDLEWARES': {}}) o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'fail'}) with pytest.raises(ImportError): _load_addons(addons, settings, o)
def test_update_settings_dont_fail_on_non_string(): test = EntrypointSettings() test.setdict({'a': 3}, 10) assert test['a'] == 3
def test_update_settings_base_test2(): test = EntrypointSettings() test.setdict({'a': 'b', 'c': 'd'}, 10) assert len(test.attributes) == 2
def test_update_settings_base_test(): test = EntrypointSettings() test.setdict({'a': 'b'}, 10) assert test['a'] == 'b'
def test_update_settings_void_dictionaries(): test = EntrypointSettings() test.setdict({}, 10) assert len(test.attributes) == 0
def test_update_settings_base_test(): test = EntrypointSettings() test.setdict({'a': 'b'}, 10) assert test['a'] == 'b'
def test_get_action_on_missing_addons_warn_if_wrong_value(): o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'wrong'}) assert _get_action_on_missing_addons(o) == 'warn'
def test_update_settings_base_test2(): test = EntrypointSettings() test.setdict({'a': 'b', 'c': 'd'}, 10) assert len(test.attributes) == 2
def test_load_addons_void(): addons = [] settings, o = EntrypointSettings(), EntrypointSettings() _load_addons(addons, settings, o) assert addons == [] assert settings.attributes == o.attributes == {}
def test_get_action_on_missing_addons_base(): o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'fail'}) assert _get_action_on_missing_addons(o) == 'fail'
def test_get_action_on_missing_addons_warn_if_wrong_value(): o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'wrong'}) assert _get_action_on_missing_addons(o) == 'warn'
def test_get_action_on_missing_addons_base(): o = EntrypointSettings() o.setdict({'ON_MISSING_ADDONS': 'fail'}) assert _get_action_on_missing_addons(o) == 'fail'
def test_get_action_on_missing_addons_default(): o = EntrypointSettings() assert _get_action_on_missing_addons(o) == 'warn'
def test_update_settings_per_key_priorities_old_behavior(): test = EntrypointSettings() test.set('ITEM_PIPELINES', {'path.one': 100}) test.set('ITEM_PIPELINES', {'path.two': 200}) assert test['ITEM_PIPELINES'] == {'path.two': 200}
def test_update_settings_void_dictionaries(): test = EntrypointSettings() test.setdict({}, 10) assert len(test.attributes) == 0
def test_update_settings_per_key_priorities_old_behavior(): test = EntrypointSettings() test.set('ITEM_PIPELINES', {'path.one': 100}) test.set('ITEM_PIPELINES', {'path.two': 200}) assert test['ITEM_PIPELINES'] == {'path.two': 200}
def test_update_settings_check_unicode_in_py3(): test = EntrypointSettings() test.setdict({'\xf1e\xf1e\xf1e': 'test'}, 10) assert test['\xf1e\xf1e\xf1e'] == 'test'
def test_update_settings_check_unicode_in_py3(): test = EntrypointSettings() test.setdict({'\xf1e\xf1e\xf1e': 'test'}, 10) assert test['\xf1e\xf1e\xf1e'] == 'test'
def test_update_settings_dont_fail_on_non_string(): test = EntrypointSettings() test.setdict({'a': 3}, 10) assert test['a'] == 3