def test_update_jsonstring(self):
settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
settings.update('{"number": 1, "newnumber": 2}')
self.assertEqual(settings['number'], 1)
self.assertEqual(settings['newnumber'], 2)
settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
self.assertEqual(settings['dict']['key'], "newval")
self.assertEqual(settings['dict']['newkey'], "newval2")
def test_update_jsonstring(self):
settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
settings.update('{"number": 1, "newnumber": 2}')
self.assertEqual(settings['number'], 1)
self.assertEqual(settings['newnumber'], 2)
settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
self.assertEqual(settings['dict']['key'], "newval")
self.assertEqual(settings['dict']['newkey'], "newval2")
def test_update(self):
settings = BaseSettings({'key_lowprio': 0}, priority=0)
settings.set('key_highprio', 10, priority=50)
custom_settings = BaseSettings({
'key_lowprio': 1,
'key_highprio': 11
},
priority=30)
custom_settings.set('newkey_one', None, priority=50)
custom_dict = {
'key_lowprio': 2,
'key_highprio': 12,
'newkey_two': None
}
settings.update(custom_dict, priority=20)
self.assertEqual(settings['key_lowprio'], 2)
self.assertEqual(settings.getpriority('key_lowprio'), 20)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_two', settings)
self.assertEqual(settings.getpriority('newkey_two'), 20)
settings.update(custom_settings)
self.assertEqual(settings['key_lowprio'], 1)
self.assertEqual(settings.getpriority('key_lowprio'), 30)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_one', settings)
self.assertEqual(settings.getpriority('newkey_one'), 50)
settings.update({'key_lowprio': 3}, priority=20)
self.assertEqual(settings['key_lowprio'], 1)
def from_crawler(cls, crawler: Crawler):
base_settings: BaseSettings = crawler.settings
cls.normalize(base_settings)
settings = BaseSettings(priority='spider')
cls.from_object(settings, crawler.spidercls.SpiderConfig)
settings.update(
{k: v
for k, v in base_settings.items() if k in settings},
priority='cmdline')
preset = base_settings.get('PRESET')
if preset:
preset_dict = BaseSettings(priority=35)
cls.from_pyfile(preset_dict, preset)
settings.update(preset_dict)
adapted = BaseSettings(priority=50)
for k, v in settings.items():
adapt = getattr(SettingsAdapter, k.lower(), None)
if adapt:
adapted.update(adapt(v))
else:
adapted[k] = v
base_settings.update(adapted.copy_to_dict(), priority=50)
base_settings['SPIDER_CONFIG'] = adapted
return cls()
def getwithbase(self, name):
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
:param name: name of the dictionary-like setting
:type name: str
"""
compbs = BaseSettings()
compbs.update(self[name + '_BASE'])
compbs.update(self[name])
compbs.update(self[name + '_USER'])
return compbs
def test_update(self):
settings = BaseSettings({'key_lowprio': 0}, priority=0)
settings.set('key_highprio', 10, priority=50)
custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11}, priority=30)
custom_settings.set('newkey_one', None, priority=50)
custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None}
settings.update(custom_dict, priority=20)
self.assertEqual(settings['key_lowprio'], 2)
self.assertEqual(settings.getpriority('key_lowprio'), 20)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_two', settings)
self.assertEqual(settings.getpriority('newkey_two'), 20)
settings.update(custom_settings)
self.assertEqual(settings['key_lowprio'], 1)
self.assertEqual(settings.getpriority('key_lowprio'), 30)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_one', settings)
self.assertEqual(settings.getpriority('newkey_one'), 50)
settings.update({'key_lowprio': 3}, priority=20)
self.assertEqual(settings['key_lowprio'], 1)
