def test_populate_settings_addons():
addon = TEST_ADDON.copy()
addon['default_settings'] = {'TEST_SETTING_A': 'by_addon'}
msg = {'enabled_addons': [addon]}
result = _populate_settings_base(msg, lambda x: x)
assert result
assert result['TEST_SETTING_A'] == 'by_addon'
def test_populate_settings_addons():
addon = TEST_ADDON.copy()
addon['default_settings'] = {'TEST_SETTING_A': 'by_addon'}
msg = {'enabled_addons': [addon]}
result = _populate_settings_base(msg, lambda x: x)
assert result
assert result['TEST_SETTING_A'] == 'by_addon'
def test_populate_settings_with_default():
def default_test(s):
s.set('TEST_SETTING_A', 'test')
result = _populate_settings_base({}, default_test)
assert result
assert result['TEST_SETTING_A'] == 'test'
def test_populate_settings_keep_user_priorities(get_settings_mock):
get_settings_mock.return_value = Settings({
'EXTENSIONS_BASE': {
'sh_scrapy.extension.HubstorageExtension': None,
'scrapy.spidermiddlewares.depth.DepthMiddleware': 10
},
'SPIDER_MIDDLEWARES_BASE': {
'scrapy.utils.misc.load_object': 1
}
})
addon = TEST_ADDON.copy()
api_settings = {
'project_settings': {
'EXTENSIONS_BASE': {
'sh_scrapy.extension.HubstorageExtension': 300,
'scrapy.contrib.throttle.AutoThrottle': 5
}
},
'enabled_addons': [addon]
}
result = _populate_settings_base(api_settings, lambda x: x, spider=True)
assert result.getdict(
'SPIDER_MIDDLEWARES_BASE')['scrapy.utils.misc.load_object'] == 1
assert result.getdict(
'EXTENSIONS_BASE')['sh_scrapy.extension.HubstorageExtension'] is None
autothrottles = [
k for k in result.getdict('EXTENSIONS_BASE') if 'AutoThrottle' in k
]
assert result.getdict('EXTENSIONS_BASE')[autothrottles[0]] == 5
def test_populate_settings_with_spider():
msg = {'project_settings': {'JOBDIR': 'by_project'},
'spider_settings': {'TEST_SETTING_A': 'test'}}
result = _populate_settings_base(msg, lambda x: x, spider=True)
assert result
assert result['TEST_SETTING_A'] == 'test'
assert result['JOBDIR'].split('/')[-1].startswith('jobdata-')
assert not result.get('PROJECT_ZIPFILE')
def test_populate_settings_override_settings():
msg = {}
for section in [
'project_settings', 'organization_settings', 'job_settings'
]:
msg[section] = {'TEST_SETTING_A': 'from_' + section}
result = _populate_settings_base(msg, lambda x: x)
assert result
assert result['TEST_SETTING_A'] == 'from_' + section
def test_populate_settings_override_settings():
msg = {}
for section in ['project_settings',
'organization_settings',
'job_settings']:
msg[section] = {'TEST_SETTING_A': 'from_' + section}
result = _populate_settings_base(msg, lambda x: x)
assert result
assert result['TEST_SETTING_A'] == 'from_' + section
def test_populate_settings_keep_user_priorities_oldpath(get_settings_mock):
get_settings_mock.return_value = Settings({
'EXTENSIONS_BASE': {'scrapy.contrib.throttle.AutoThrottle': 0}})
api_settings = {
'project_settings': {
'EXTENSIONS_BASE': {'scrapy.contrib.throttle.AutoThrottle': 5}}}
result = _populate_settings_base(api_settings, lambda x: x, spider=True)
autothrottles = [k for k in result.getdict('EXTENSIONS_BASE')
if 'AutoThrottle' in k]
assert len(autothrottles) == 1
assert result.getdict('EXTENSIONS_BASE')[autothrottles[0]] is 0
def test_populate_settings_with_spider():
msg = {
'project_settings': {
'JOBDIR': 'by_project'
},
'spider_settings': {
'TEST_SETTING_A': 'test'
}
}
result = _populate_settings_base(msg, lambda x: x, spider=True)
assert result
assert result['TEST_SETTING_A'] == 'test'
assert result['JOBDIR'].split('/')[-1].startswith('jobdata-')
assert not result.get('PROJECT_ZIPFILE')
def test_populate_settings_keep_user_priorities_oldpath(get_settings_mock):
get_settings_mock.return_value = Settings(
{'EXTENSIONS_BASE': {
'scrapy.contrib.throttle.AutoThrottle': 0
}})
api_settings = {
'project_settings': {
'EXTENSIONS_BASE': {
'scrapy.contrib.throttle.AutoThrottle': 5
}
}
}
result = _populate_settings_base(api_settings, lambda x: x, spider=True)
autothrottles = [
k for k in result.getdict('EXTENSIONS_BASE') if 'AutoThrottle' in k
]
assert len(autothrottles) == 1
assert result.getdict('EXTENSIONS_BASE')[autothrottles[0]] is 0
def test_populate_settings_keep_user_priorities(get_settings_mock):
get_settings_mock.return_value = Settings({
'EXTENSIONS_BASE': {
'sh_scrapy.extension.HubstorageExtension': None,
'scrapy.spidermiddlewares.depth.DepthMiddleware': 10},
'SPIDER_MIDDLEWARES_BASE': {'scrapy.utils.misc.load_object': 1}})
addon = TEST_ADDON.copy()
api_settings = {
'project_settings': {
'EXTENSIONS_BASE': {'sh_scrapy.extension.HubstorageExtension': 300,
'scrapy.contrib.throttle.AutoThrottle': 5}},
'enabled_addons': [addon]}
result = _populate_settings_base(api_settings, lambda x: x, spider=True)
assert result.getdict('SPIDER_MIDDLEWARES_BASE')[
'scrapy.utils.misc.load_object'] == 1
assert result.getdict('EXTENSIONS_BASE')[
'sh_scrapy.extension.HubstorageExtension'] is None
autothrottles = [k for k in result.getdict('EXTENSIONS_BASE')
if 'AutoThrottle' in k]
assert result.getdict('EXTENSIONS_BASE')[autothrottles[0]] == 5
def test_populate_settings_with_spider_autoscraping():
result = _populate_settings_base({}, lambda x: x, spider=True)
assert result
assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip'
def test_populate_settings_with_spider_override():
msg = {'job_settings': {'JOBDIR': 'by_job'}}
result = _populate_settings_base(msg, lambda x: x, spider=True)
assert result
assert result['JOBDIR'] == 'by_job'
def test_populate_settings_with_spider_override():
msg = {'job_settings': {'JOBDIR': 'by_job'}}
result = _populate_settings_base(msg, lambda x: x, spider=True)
assert result
assert result['JOBDIR'] == 'by_job'
def test_populate_settings_with_spider_autoscraping():
result = _populate_settings_base({}, lambda x: x, spider=True)
assert result
assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip'
def test_populate_settings_with_default():
def default_test(s):
s.set('TEST_SETTING_A', 'test')
result = _populate_settings_base({}, default_test)
assert result
assert result['TEST_SETTING_A'] == 'test'
def test_populate_settings_unique_update_dict():
monitoring_dict = {u'SPIDER_OPENED': {u'failed_actions': []}}
msg = {'spider_settings': {'DASH_MONITORING': monitoring_dict}}
result = _populate_settings_base(msg, lambda x: x, spider=True)
assert result['DASH_MONITORING'] == monitoring_dict
def test_populate_settings_unique_update_dict():
monitoring_dict = {u'SPIDER_OPENED': {u'failed_actions': []}}
msg = {'spider_settings': {'DASH_MONITORING': monitoring_dict}}
result = _populate_settings_base(msg, lambda x: x, spider=True)
assert result['DASH_MONITORING'] == monitoring_dict
def test_populate_settings_dont_fail():
result = _populate_settings_base({}, lambda x: x)
assert isinstance(result, Settings)
def test_populate_settings_dont_fail():
result = _populate_settings_base({}, lambda x: x)
assert isinstance(result, Settings)
