def test_update_settings_check_unicode_in_py2_key_value():
# a dict entry is duplicated as unicode doesn't match native str value
test = EntrypointSettings()
test.setdict({'\xf1e\xf1e\xf1e': '\xf1e\xf1e'}, 10)
assert test['\xf1e\xf1e\xf1e'] == '\xf1e\xf1e'
native_key = to_native_str('\xf1e\xf1e\xf1e')
assert test[native_key] == to_native_str('\xf1e\xf1e')
def test_update_settings_check_unicode_in_py2_key_value():
# a dict entry is duplicated as unicode doesn't match native str value
test = EntrypointSettings()
test.setdict({'\xf1e\xf1e\xf1e': '\xf1e\xf1e'}, 10)
assert test['\xf1e\xf1e\xf1e'] == '\xf1e\xf1e'
native_key = to_native_str('\xf1e\xf1e\xf1e')
assert test[native_key] == to_native_str('\xf1e\xf1e')
def test_load_addons_basic_usage():
addons = [TEST_ADDON]
settings = EntrypointSettings()
settings.setdict({'SPIDER_MIDDLEWARES': {}})
o = EntrypointSettings()
_load_addons(addons, settings, o)
assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
def test_load_addons_basic_usage():
addons = [TEST_ADDON]
settings = EntrypointSettings()
settings.setdict({'SPIDER_MIDDLEWARES': {}})
o = EntrypointSettings()
_load_addons(addons, settings, o)
assert settings['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
assert o['SPIDER_MIDDLEWARES'] == {TEST_ADDON['path']: 10}
def test_update_settings_update_existing_value():
test = EntrypointSettings()
test.setdict({'a': 'b', 'c': 'd'}, priority=10)
test.setdict({'c': 'e', 'f': 'g'}, 10)
assert len(test.attributes) == 3
assert test['a'] == 'b'
assert test['c'] == 'e'
assert test['f'] == 'g'
def test_update_settings_update_existing_value():
test = EntrypointSettings()
test.setdict({'a': 'b', 'c': 'd'}, priority=10)
test.setdict({'c': 'e', 'f': 'g'}, 10)
assert len(test.attributes) == 3
assert test['a'] == 'b'
assert test['c'] == 'e'
assert test['f'] == 'g'
def test_load_addons_hworker_fail_on_import():
addons = [TEST_ADDON.copy()]
addons[0]['path'] = 'hworker.some.module'
settings = EntrypointSettings()
settings.setdict({'SPIDER_MIDDLEWARES': {}})
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'fail'})
with pytest.raises(ImportError):
_load_addons(addons, settings, o)
def test_load_addons_hworker_fail_on_import():
addons = [TEST_ADDON.copy()]
addons[0]['path'] = 'hworker.some.module'
settings = EntrypointSettings()
settings.setdict({'SPIDER_MIDDLEWARES': {}})
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'fail'})
with pytest.raises(ImportError):
_load_addons(addons, settings, o)
def test_load_addons_hworker_warning_on_import():
addons = [TEST_ADDON.copy()]
addons[0]['path'] = 'hworker.some.module'
settings = {'SPIDER_MIDDLEWARES': {}}
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'warn'})
_load_addons(addons, settings, o)
assert len(o.attributes) == 1
assert o['ON_MISSING_ADDONS'] == 'warn'
assert settings == {'SPIDER_MIDDLEWARES': {}}
def test_maybe_load_autoscraping_project_ok():
result = EntrypointSettings()
result.setdict({'SPIDER_MANAGER_CLASS': 'test.class'})
_maybe_load_autoscraping_project(result)
assert result['ITEM_PIPELINES'] == {
'slybot.dupefilter.DupeFilterPipeline': 0}
assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip'
assert result['SLYCLOSE_SPIDER_ENABLED']
assert result['SLYDUPEFILTER_ENABLED']
assert result['SPIDER_MANAGER_CLASS'] == 'test.class'
def test_load_addons_hworker_warning_on_import():
addons = [TEST_ADDON.copy()]
addons[0]['path'] = 'hworker.some.module'
settings = {'SPIDER_MIDDLEWARES': {}}
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'warn'})
_load_addons(addons, settings, o)
assert len(o.attributes) == 1
assert o['ON_MISSING_ADDONS'] == 'warn'
assert settings == {'SPIDER_MIDDLEWARES': {}}
def test_maybe_load_autoscraping_project_ok():
result = EntrypointSettings()
result.setdict({'SPIDER_MANAGER_CLASS': 'test.class'})
_maybe_load_autoscraping_project(result)
assert result['ITEM_PIPELINES'] == {
'slybot.dupefilter.DupeFilterPipeline': 0
}
assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip'
assert result['SLYCLOSE_SPIDER_ENABLED']
assert result['SLYDUPEFILTER_ENABLED']
assert result['SPIDER_MANAGER_CLASS'] == 'test.class'
def test_load_addons_basic_with_defaults():
addons = [TEST_ADDON.copy()]
addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'}
settings = {'SPIDER_MIDDLEWARES_BASE': {
'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500}}
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'warn'})
_load_addons(addons, settings, o)
assert settings == {'SPIDER_MIDDLEWARES_BASE': {
TEST_ADDON['path']: 10,
'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
}}
assert len(o.attributes) == 3
assert o['TEST_SETTING_A'] == 'TEST'
assert o['ON_MISSING_ADDONS'] == 'warn'
assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3
def test_load_addons_basic_with_defaults():
addons = [TEST_ADDON.copy()]
addons[0]['default_settings'] = {'TEST_SETTING_A': 'TEST'}
settings = {
'SPIDER_MIDDLEWARES_BASE': {
'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
}
}
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'warn'})
_load_addons(addons, settings, o)
assert settings == {
'SPIDER_MIDDLEWARES_BASE': {
TEST_ADDON['path']: 10,
'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500
}
}
assert len(o.attributes) == 3
assert o['TEST_SETTING_A'] == 'TEST'
assert o['ON_MISSING_ADDONS'] == 'warn'
assert len(o['SPIDER_MIDDLEWARES_BASE']) == 3
def test_update_settings_base_test():
test = EntrypointSettings()
test.setdict({'a': 'b'}, 10)
assert test['a'] == 'b'
def test_update_settings_void_dictionaries():
test = EntrypointSettings()
test.setdict({}, 10)
assert len(test.attributes) == 0
def test_update_settings_void_dictionaries():
test = EntrypointSettings()
test.setdict({}, 10)
assert len(test.attributes) == 0
def test_get_action_on_missing_addons_warn_if_wrong_value():
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'wrong'})
assert _get_action_on_missing_addons(o) == 'warn'
def test_get_action_on_missing_addons_base():
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'fail'})
assert _get_action_on_missing_addons(o) == 'fail'
def test_get_action_on_missing_addons_warn_if_wrong_value():
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'wrong'})
assert _get_action_on_missing_addons(o) == 'warn'
def test_update_settings_base_test2():
test = EntrypointSettings()
test.setdict({'a': 'b', 'c': 'd'}, 10)
assert len(test.attributes) == 2
def test_update_settings_base_test():
test = EntrypointSettings()
test.setdict({'a': 'b'}, 10)
assert test['a'] == 'b'
def test_update_settings_dont_fail_on_non_string():
test = EntrypointSettings()
test.setdict({'a': 3}, 10)
assert test['a'] == 3
def test_update_settings_base_test2():
test = EntrypointSettings()
test.setdict({'a': 'b', 'c': 'd'}, 10)
assert len(test.attributes) == 2
def test_update_settings_dont_fail_on_non_string():
test = EntrypointSettings()
test.setdict({'a': 3}, 10)
assert test['a'] == 3
def test_get_action_on_missing_addons_base():
o = EntrypointSettings()
o.setdict({'ON_MISSING_ADDONS': 'fail'})
assert _get_action_on_missing_addons(o) == 'fail'
def test_update_settings_check_unicode_in_py3():
test = EntrypointSettings()
test.setdict({'\xf1e\xf1e\xf1e': 'test'}, 10)
assert test['\xf1e\xf1e\xf1e'] == 'test'
def test_update_settings_check_unicode_in_py3():
test = EntrypointSettings()
test.setdict({'\xf1e\xf1e\xf1e': 'test'}, 10)
assert test['\xf1e\xf1e\xf1e'] == 'test'