def from_crawler(cls, crawler: Crawler): base_settings: BaseSettings = crawler.settings cls.normalize(base_settings) settings = BaseSettings(priority='spider') cls.from_object(settings, crawler.spidercls.SpiderConfig) settings.update( {k: v for k, v in base_settings.items() if k in settings}, priority='cmdline') preset = base_settings.get('PRESET') if preset: preset_dict = BaseSettings(priority=35) cls.from_pyfile(preset_dict, preset) settings.update(preset_dict) adapted = BaseSettings(priority=50) for k, v in settings.items(): adapt = getattr(SettingsAdapter, k.lower(), None) if adapt: adapted.update(adapt(v)) else: adapted[k] = v base_settings.update(adapted.copy_to_dict(), priority=50) base_settings['SPIDER_CONFIG'] = adapted return cls()
def test_copy_to_dict(self): s = BaseSettings({ 'TEST_STRING': 'a string', 'TEST_LIST': [1, 2], 'TEST_BOOLEAN': False, 'TEST_BASE': BaseSettings({ 1: 1, 2: 2 }, 'project'), 'TEST': BaseSettings({ 1: 10, 3: 30 }, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default') }) self.assertDictEqual( s.copy_to_dict(), { 'HASNOBASE': { 3: 3000 }, 'TEST': { 1: 10, 3: 30 }, 'TEST_BASE': { 1: 1, 2: 2 }, 'TEST_BOOLEAN': False, 'TEST_LIST': [1, 2], 'TEST_STRING': 'a string' })
def test_copy_to_dict(self): s = BaseSettings({'TEST_STRING': 'a string', 'TEST_LIST': [1, 2], 'TEST_BOOLEAN': False, 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) self.assertDictEqual(s.copy_to_dict(), {'HASNOBASE': {3: 3000}, 'TEST': {1: 10, 3: 30}, 'TEST_BASE': {1: 1, 2: 2}, 'TEST_BOOLEAN': False, 'TEST_LIST': [1, 2], 'TEST_STRING': 'a string'})