def test_populate_settings_addons(): addon = TEST_ADDON.copy() addon['default_settings'] = {'TEST_SETTING_A': 'by_addon'} msg = {'enabled_addons': [addon]} result = _populate_settings_base(msg, lambda x: x) assert result assert result['TEST_SETTING_A'] == 'by_addon'
def test_populate_settings_addons(): addon = TEST_ADDON.copy() addon['default_settings'] = {'TEST_SETTING_A': 'by_addon'} msg = {'enabled_addons': [addon]} result = _populate_settings_base(msg, lambda x: x) assert result assert result['TEST_SETTING_A'] == 'by_addon'
def test_populate_settings_with_default(): def default_test(s): s.set('TEST_SETTING_A', 'test') result = _populate_settings_base({}, default_test) assert result assert result['TEST_SETTING_A'] == 'test'
def test_populate_settings_keep_user_priorities(get_settings_mock): get_settings_mock.return_value = Settings({ 'EXTENSIONS_BASE': { 'sh_scrapy.extension.HubstorageExtension': None, 'scrapy.spidermiddlewares.depth.DepthMiddleware': 10 }, 'SPIDER_MIDDLEWARES_BASE': { 'scrapy.utils.misc.load_object': 1 } }) addon = TEST_ADDON.copy() api_settings = { 'project_settings': { 'EXTENSIONS_BASE': { 'sh_scrapy.extension.HubstorageExtension': 300, 'scrapy.contrib.throttle.AutoThrottle': 5 } }, 'enabled_addons': [addon] } result = _populate_settings_base(api_settings, lambda x: x, spider=True) assert result.getdict( 'SPIDER_MIDDLEWARES_BASE')['scrapy.utils.misc.load_object'] == 1 assert result.getdict( 'EXTENSIONS_BASE')['sh_scrapy.extension.HubstorageExtension'] is None autothrottles = [ k for k in result.getdict('EXTENSIONS_BASE') if 'AutoThrottle' in k ] assert result.getdict('EXTENSIONS_BASE')[autothrottles[0]] == 5
def test_populate_settings_with_spider(): msg = {'project_settings': {'JOBDIR': 'by_project'}, 'spider_settings': {'TEST_SETTING_A': 'test'}} result = _populate_settings_base(msg, lambda x: x, spider=True) assert result assert result['TEST_SETTING_A'] == 'test' assert result['JOBDIR'].split('/')[-1].startswith('jobdata-') assert not result.get('PROJECT_ZIPFILE')
def test_populate_settings_override_settings(): msg = {} for section in [ 'project_settings', 'organization_settings', 'job_settings' ]: msg[section] = {'TEST_SETTING_A': 'from_' + section} result = _populate_settings_base(msg, lambda x: x) assert result assert result['TEST_SETTING_A'] == 'from_' + section
def test_populate_settings_override_settings(): msg = {} for section in ['project_settings', 'organization_settings', 'job_settings']: msg[section] = {'TEST_SETTING_A': 'from_' + section} result = _populate_settings_base(msg, lambda x: x) assert result assert result['TEST_SETTING_A'] == 'from_' + section
def test_populate_settings_keep_user_priorities_oldpath(get_settings_mock): get_settings_mock.return_value = Settings({ 'EXTENSIONS_BASE': {'scrapy.contrib.throttle.AutoThrottle': 0}}) api_settings = { 'project_settings': { 'EXTENSIONS_BASE': {'scrapy.contrib.throttle.AutoThrottle': 5}}} result = _populate_settings_base(api_settings, lambda x: x, spider=True) autothrottles = [k for k in result.getdict('EXTENSIONS_BASE') if 'AutoThrottle' in k] assert len(autothrottles) == 1 assert result.getdict('EXTENSIONS_BASE')[autothrottles[0]] is 0
def test_populate_settings_with_spider(): msg = { 'project_settings': { 'JOBDIR': 'by_project' }, 'spider_settings': { 'TEST_SETTING_A': 'test' } } result = _populate_settings_base(msg, lambda x: x, spider=True) assert result assert result['TEST_SETTING_A'] == 'test' assert result['JOBDIR'].split('/')[-1].startswith('jobdata-') assert not result.get('PROJECT_ZIPFILE')
def test_populate_settings_keep_user_priorities_oldpath(get_settings_mock): get_settings_mock.return_value = Settings( {'EXTENSIONS_BASE': { 'scrapy.contrib.throttle.AutoThrottle': 0 }}) api_settings = { 'project_settings': { 'EXTENSIONS_BASE': { 'scrapy.contrib.throttle.AutoThrottle': 5 } } } result = _populate_settings_base(api_settings, lambda x: x, spider=True) autothrottles = [ k for k in result.getdict('EXTENSIONS_BASE') if 'AutoThrottle' in k ] assert len(autothrottles) == 1 assert result.getdict('EXTENSIONS_BASE')[autothrottles[0]] is 0
def test_populate_settings_keep_user_priorities(get_settings_mock): get_settings_mock.return_value = Settings({ 'EXTENSIONS_BASE': { 'sh_scrapy.extension.HubstorageExtension': None, 'scrapy.spidermiddlewares.depth.DepthMiddleware': 10}, 'SPIDER_MIDDLEWARES_BASE': {'scrapy.utils.misc.load_object': 1}}) addon = TEST_ADDON.copy() api_settings = { 'project_settings': { 'EXTENSIONS_BASE': {'sh_scrapy.extension.HubstorageExtension': 300, 'scrapy.contrib.throttle.AutoThrottle': 5}}, 'enabled_addons': [addon]} result = _populate_settings_base(api_settings, lambda x: x, spider=True) assert result.getdict('SPIDER_MIDDLEWARES_BASE')[ 'scrapy.utils.misc.load_object'] == 1 assert result.getdict('EXTENSIONS_BASE')[ 'sh_scrapy.extension.HubstorageExtension'] is None autothrottles = [k for k in result.getdict('EXTENSIONS_BASE') if 'AutoThrottle' in k] assert result.getdict('EXTENSIONS_BASE')[autothrottles[0]] == 5
def test_populate_settings_with_spider_autoscraping(): result = _populate_settings_base({}, lambda x: x, spider=True) assert result assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip'
def test_populate_settings_with_spider_override(): msg = {'job_settings': {'JOBDIR': 'by_job'}} result = _populate_settings_base(msg, lambda x: x, spider=True) assert result assert result['JOBDIR'] == 'by_job'
def test_populate_settings_with_spider_override(): msg = {'job_settings': {'JOBDIR': 'by_job'}} result = _populate_settings_base(msg, lambda x: x, spider=True) assert result assert result['JOBDIR'] == 'by_job'
def test_populate_settings_with_spider_autoscraping(): result = _populate_settings_base({}, lambda x: x, spider=True) assert result assert result['PROJECT_ZIPFILE'] == 'project-slybot.zip'
def test_populate_settings_with_default(): def default_test(s): s.set('TEST_SETTING_A', 'test') result = _populate_settings_base({}, default_test) assert result assert result['TEST_SETTING_A'] == 'test'
def test_populate_settings_unique_update_dict(): monitoring_dict = {u'SPIDER_OPENED': {u'failed_actions': []}} msg = {'spider_settings': {'DASH_MONITORING': monitoring_dict}} result = _populate_settings_base(msg, lambda x: x, spider=True) assert result['DASH_MONITORING'] == monitoring_dict
def test_populate_settings_unique_update_dict(): monitoring_dict = {u'SPIDER_OPENED': {u'failed_actions': []}} msg = {'spider_settings': {'DASH_MONITORING': monitoring_dict}} result = _populate_settings_base(msg, lambda x: x, spider=True) assert result['DASH_MONITORING'] == monitoring_dict
def test_populate_settings_dont_fail(): result = _populate_settings_base({}, lambda x: x) assert isinstance(result, Settings)
def test_populate_settings_dont_fail(): result = _populate_settings_base({}, lambda x: x) assert isinstance(result, Settings)