def test_update_jsonstring(self):
settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
settings.update('{"number": 1, "newnumber": 2}')
self.assertEqual(settings['number'], 1)
self.assertEqual(settings['newnumber'], 2)
settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
self.assertEqual(settings['dict']['key'], "newval")
self.assertEqual(settings['dict']['newkey'], "newval2")
def test_getwithbase(self):
s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
s['TEST'].set(2, 200, 'cmdline')
six.assertCountEqual(self, s.getwithbase('TEST'),
{1: 1, 2: 200, 3: 30})
six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
self.assertEqual(s.getwithbase('NONEXISTENT'), {})
def test_setitem(self):
settings = BaseSettings()
settings.set('key', 'a', 'default')
settings['key'] = 'b'
self.assertEqual(settings['key'], 'b')
self.assertEqual(settings.getpriority('key'), 20)
settings['key'] = 'c'
self.assertEqual(settings['key'], 'c')
settings['key2'] = 'x'
self.assertIn('key2', settings)
self.assertEqual(settings['key2'], 'x')
self.assertEqual(settings.getpriority('key2'), 20)
def test_copy_to_dict(self):
s = BaseSettings({'TEST_STRING': 'a string',
'TEST_LIST': [1, 2],
'TEST_BOOLEAN': False,
'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
self.assertDictEqual(s.copy_to_dict(),
{'HASNOBASE': {3: 3000},
'TEST': {1: 10, 3: 30},
'TEST_BASE': {1: 1, 2: 2},
'TEST_BOOLEAN': False,
'TEST_LIST': [1, 2],
'TEST_STRING': 'a string'})
def _map_keys(compdict):
if isinstance(compdict, BaseSettings):
compbs = BaseSettings()
for k, v in compdict.items():
prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same '
'object, please update your settings'
''.format(list(compdict.keys())))
else:
compbs.set(convert(k), v, priority=prio)
return compbs
else:
_check_components(compdict)
return {convert(k): v for k, v in compdict.items()}
def test_set_per_key_priorities(self):
attribute = SettingsAttribute(
BaseSettings({'one': 10, 'two': 20}, 0), 0)
new_dict = {'one': 11, 'two': 21}
attribute.set(new_dict, 10)
self.assertEqual(attribute.value['one'], 11)
self.assertEqual(attribute.value['two'], 21)
new_settings = BaseSettings()
new_settings.set('one', 12, 20)
new_settings.set('two', 12, 0)
attribute.set(new_settings, 0)
self.assertEqual(attribute.value['one'], 12)
self.assertEqual(attribute.value['two'], 21)
def _map_keys(compdict):
if isinstance(compdict, BaseSettings):
compbs = BaseSettings()
for k, v in six.iteritems(compdict):
prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same '
'object, please update your settings'
''.format(list(compdict.keys())))
else:
compbs.set(convert(k), v, priority=prio)
return compbs
else:
_check_components(compdict)
return {convert(k): v for k, v in six.iteritems(compdict)}
def test_repr(self):
settings = BaseSettings()
self.assertEqual(repr(settings), "<BaseSettings {}>")
attr = SettingsAttribute('testval', 15)
settings['testkey'] = attr
self.assertEqual(repr(settings),
"<BaseSettings {'testkey': %s}>" % repr(attr))
def test_overwrite_basesettings(self):
original_dict = {'one': 10, 'two': 20}
original_settings = BaseSettings(original_dict, 0)
attribute = SettingsAttribute(original_settings, 0)
new_dict = {'three': 11, 'four': 21}
attribute.set(new_dict, 10)
self.assertIsInstance(attribute.value, BaseSettings)
six.assertCountEqual(self, attribute.value, new_dict)
six.assertCountEqual(self, original_settings, original_dict)
new_settings = BaseSettings({'five': 12}, 0)
attribute.set(new_settings, 0) # Insufficient priority
six.assertCountEqual(self, attribute.value, new_dict)
attribute.set(new_settings, 10)
six.assertCountEqual(self, attribute.value, new_settings)
def test_delete(self):
settings = BaseSettings({'key': None})
settings.set('key_highprio', None, priority=50)
settings.delete('key')
settings.delete('key_highprio')
self.assertNotIn('key', settings)
self.assertIn('key_highprio', settings)
del settings['key_highprio']
self.assertNotIn('key_highprio', settings)
def test_getwithbase(self):
s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
s['TEST'].set(2, 200, 'cmdline')
six.assertCountEqual(self, s.getwithbase('TEST'),
{1: 1, 2: 200, 3: 30})
six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
self.assertEqual(s.getwithbase('NONEXISTENT'), {})
def __init__(self, values=None, priority='project'):
super().__init__()
self.setmodule(aio_settings, 'default')
for name, val in self.items():
if isinstance(val, dict):
self.set(name, BaseSettings(val, 'default'), 'default')
self.update(values, priority)
def test_getcomposite(self):
s = BaseSettings({
'TEST_BASE': {
1: 1,
2: 2
},
'TEST': BaseSettings({
1: 10,
3: 30
}, 'default'),
'HASNOBASE': BaseSettings({1: 1}, 'default')
})
s['TEST'].set(4, 4, priority='project')
# When users specify a _BASE setting they explicitly don't want to use
# Scrapy's defaults, so we don't want to see anything that has a
# 'default' priority from TEST
cs = s._getcomposite('TEST')
self.assertEqual(len(cs), 3)
self.assertEqual(cs[1], 1)
self.assertEqual(cs[2], 2)
self.assertEqual(cs[4], 4)
cs = s._getcomposite('HASNOBASE')
self.assertEqual(len(cs), 1)
self.assertEqual(cs[1], 1)
cs = s._getcomposite('NONEXISTENT')
self.assertIsNone(cs)
def test_getcomposite(self):
s = BaseSettings({'TEST_BASE': {1: 1, 2: 2},
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({1: 1}, 'default')})
s['TEST'].set(4, 4, priority='project')
# When users specify a _BASE setting they explicitly don't want to use
# Scrapy's defaults, so we don't want to see anything that has a
# 'default' priority from TEST
cs = s._getcomposite('TEST')
self.assertEqual(len(cs), 3)
self.assertEqual(cs[1], 1)
self.assertEqual(cs[2], 2)
self.assertEqual(cs[4], 4)
cs = s._getcomposite('HASNOBASE')
self.assertEqual(len(cs), 1)
self.assertEqual(cs[1], 1)
cs = s._getcomposite('NONEXISTENT')
self.assertIsNone(cs)
def setUp(self):
self.settings = BaseSettings({
'S3PIPELINE_URL':
's3://my-bucket/{name}/{time}/items.{chunk:07d}.jl.gz',
'KINESISSTREAM_NAME':
'kinesis-stream',
'KENISISPARTITION_KEY':
'kinesis-partition-key'
})
def test_update_jsonstring(self):
settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
settings.update('{"number": 1, "newnumber": 2}')
self.assertEqual(settings['number'], 1)
self.assertEqual(settings['newnumber'], 2)
settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
self.assertEqual(settings['dict']['key'], "newval")
self.assertEqual(settings['dict']['newkey'], "newval2")
def test_update_settings_per_key_priorities_new_behaviour():
from scrapy.settings import BaseSettings
test = EntrypointSettings()
test.set('ITEM_PIPELINES', BaseSettings())
test['ITEM_PIPELINES'].update({'test.path1': 100})
test['ITEM_PIPELINES'].update({'test.path2': 200})
assert dict(test['ITEM_PIPELINES']) == {
'test.path1': 100,
'test.path2': 200
}
def getwithbase(self, name):
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
:param name: name of the dictionary-like setting
:type name: str
"""
compbs = BaseSettings()
compbs.update(self[name + '_BASE'])
compbs.update(self[name])
compbs.update(self[name + '_USER'])
return compbs
def test_delete(self):
settings = BaseSettings({'key': None})
settings.set('key_highprio', None, priority=50)
settings.delete('key')
settings.delete('key_highprio')
self.assertNotIn('key', settings)
self.assertIn('key_highprio', settings)
del settings['key_highprio']
self.assertNotIn('key_highprio', settings)
def test_json_gz(self):
settings = BaseSettings({
'S3PIPELINE_URL':
's3://my-bucket/{name}/{time}/items.{chunk:07d}.json.gz',
'FEED_EXPORTERS_BASE':
default_settings.FEED_EXPORTERS_BASE,
})
pipeline = S3Pipeline(settings, None)
self.assertTrue(pipeline.use_gzip)
self.assertEqual(pipeline.exporter_cls, JsonItemExporter)
def test_copy_to_dict(self):
s = BaseSettings({
'TEST_STRING': 'a string',
'TEST_LIST': [1, 2],
'TEST_BOOLEAN': False,
'TEST_BASE': BaseSettings({
1: 1,
2: 2
}, 'project'),
'TEST': BaseSettings({
1: 10,
3: 30
}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')
})
self.assertDictEqual(
s.copy_to_dict(), {
'HASNOBASE': {
3: 3000
},
'TEST': {
1: 10,
3: 30
},
'TEST_BASE': {
1: 1,
2: 2
},
'TEST_BOOLEAN': False,
'TEST_LIST': [1, 2],
'TEST_STRING': 'a string'
})
def test_force_no_gzip(self):
settings = BaseSettings({
'S3PIPELINE_URL':
's3://my-bucket/{name}/{time}/items.{chunk:07d}.jl.gz',
'S3PIPELINE_GZIP':
False,
'FEED_EXPORTERS_BASE':
default_settings.FEED_EXPORTERS_BASE,
})
pipeline = S3Pipeline(settings, None)
self.assertFalse(pipeline.use_gzip)
def test_max_wait_upload_time(self):
settings = BaseSettings({
'S3PIPELINE_URL':
's3://my-bucket/{name}/{time}/items.{chunk:07d}.jl.gz',
'S3PIPELINE_MAX_WAIT_UPLOAD_TIME':
300,
'FEED_EXPORTERS_BASE':
default_settings.FEED_EXPORTERS_BASE,
})
pipeline = S3Pipeline(settings, None)
self.assertEqual(pipeline.max_wait_upload_time, 300)
def test_max_chunk_size(self):
settings = BaseSettings({
'S3PIPELINE_URL':
's3://my-bucket/{name}/{time}/items.{chunk:07d}.jl.gz',
'S3PIPELINE_MAX_CHUNK_SIZE':
1000,
'FEED_EXPORTERS_BASE':
default_settings.FEED_EXPORTERS_BASE,
})
pipeline = S3Pipeline(settings, None)
self.assertEqual(pipeline.max_chunk_size, 1000)
def test_duplicate_components_in_basesettings(self):
# Higher priority takes precedence
duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0)
duplicate_bs.set('ONE', 4, priority=10)
self.assertEqual(build_component_list(duplicate_bs,
convert=lambda x: x.lower()),
['two', 'one'])
duplicate_bs.set('one', duplicate_bs['one'], priority=20)
self.assertEqual(build_component_list(duplicate_bs,
convert=lambda x: x.lower()),
['one', 'two'])
# Same priority raises ValueError
duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20)
self.assertRaises(ValueError, build_component_list, duplicate_bs,
convert=lambda x: x.lower())
def close(spider, reason):
settings = BaseSettings({
'MAIL_FROM': '*****@*****.**',
'MAIL_HOST': 'smtp.aliyun.com',
'MAIL_PORT': '25',
'MAIL_USER': '******',
'MAIL_PASS': '******',
})
print 'start send email'
mailer = MailSender.from_settings(settings=settings)
mailer.send(to=["*****@*****.**"],
subject="job spider end",
body=reason)
print 'end send email'
def test_duplicate_components_in_basesettings(self):
# Higher priority takes precedence
duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0)
duplicate_bs.set('ONE', 4, priority=10)
self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()),
['two', 'one'])
duplicate_bs.set('one', duplicate_bs['one'], priority=20)
self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()),
['one', 'two'])
# Same priority raises ValueError
duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20)
self.assertRaises(ValueError,
build_component_list, duplicate_bs, convert=lambda x: x.lower())
def test_gcs(self):
settings = BaseSettings({
'S3PIPELINE_URL':
'gs://my-bucket/{name}/{time}/items.{chunk:07d}.jl',
'FEED_EXPORTERS_BASE':
default_settings.FEED_EXPORTERS_BASE,
})
pipeline = S3Pipeline(settings, None)
self.assertEqual(pipeline.bucket_name, 'my-bucket')
self.assertEqual(pipeline.object_key_template,
'{name}/{time}/items.{chunk:07d}.jl')
self.assertEqual(pipeline.max_chunk_size, 100)
self.assertFalse(pipeline.use_gzip)
self.assertEqual(pipeline.max_wait_upload_time, 30)
self.assertIsInstance(pipeline.strategy, GCSStrategy)
self.assertEqual(pipeline.exporter_cls, JsonLinesItemExporter)
def from_settings(cls: Type[RFPDupeFilterTV],
settings: BaseSettings,
*,
fingerprinter=None) -> RFPDupeFilterTV:
debug = settings.getbool('DUPEFILTER_DEBUG')
try:
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
except TypeError:
warn(
"RFPDupeFilter subclasses must either modify their '__init__' "
"method to support a 'fingerprinter' parameter or reimplement "
"the 'from_settings' class method.",
ScrapyDeprecationWarning,
)
result = cls(job_dir(settings), debug)
result.fingerprinter = fingerprinter
return result
def test_setitem(self):
settings = BaseSettings()
settings.set('key', 'a', 'default')
settings['key'] = 'b'
self.assertEqual(settings['key'], 'b')
self.assertEqual(settings.getpriority('key'), 20)
settings['key'] = 'c'
self.assertEqual(settings['key'], 'c')
settings['key2'] = 'x'
self.assertIn('key2', settings)
self.assertEqual(settings['key2'], 'x')
self.assertEqual(settings.getpriority('key2'), 20)
def test_duplicate_components_in_basesettings(self):
# Higher priority takes precedence
duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0)
duplicate_bs.set("ONE", 4, priority=10)
self.assertEqual(
build_component_list(duplicate_bs, convert=lambda x: x.lower()),
["two", "one"],
)
duplicate_bs.set("one", duplicate_bs["one"], priority=20)
self.assertEqual(
build_component_list(duplicate_bs, convert=lambda x: x.lower()),
["one", "two"],
)
# Same priority raises ValueError
duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)
self.assertRaises(ValueError,
build_component_list,
duplicate_bs,
convert=lambda x: x.lower())
def spider_opened(self, spider):
try:
spider_attr = getattr(spider, "TOR_PROXY_ENABLED")
except AttributeError:
if not spider.crawler.settings.getbool("TOR_PROXY_ENABLED"):
self.enabled = False
self.logger.info("Tor Proxy disabled (TOR_PROXY_ENABLED setting)")
return
else:
if not BaseSettings({"enabled": spider_attr}).getbool("enabled"):
self.enabled = False
self.logger.info("Tor Proxy disabled (tor_proxy_enabled spider attribute)")
return
self.enabled = True
self._read_settings(spider.crawler.settings)
if self.enabled:
self.logger.info("Using Tor Proxy at %s", self.proxy_url)
def test_set_per_key_priorities(self):
attribute = SettingsAttribute(BaseSettings({
'one': 10,
'two': 20
}, 0), 0)
new_dict = {'one': 11, 'two': 21}
attribute.set(new_dict, 10)
self.assertEqual(attribute.value['one'], 11)
self.assertEqual(attribute.value['two'], 21)
new_settings = BaseSettings()
new_settings.set('one', 12, 20)
new_settings.set('two', 12, 0)
attribute.set(new_settings, 0)
self.assertEqual(attribute.value['one'], 12)
self.assertEqual(attribute.value['two'], 21)
def spider_opened(self, spider):
try:
spider_attr = getattr(spider, "crawlera_fetch_enabled")
except AttributeError:
if not spider.crawler.settings.getbool("CRAWLERA_FETCH_ENABLED"):
self.enabled = False
logger.info(
"Crawlera Fetch disabled (CRAWLERA_FETCH_ENABLED setting)")
return
else:
if not BaseSettings({"enabled": spider_attr}).getbool("enabled"):
self.enabled = False
logger.info(
"Crawlera Fetch disabled (crawlera_fetch_enabled spider attribute)"
)
return
self.enabled = True
self._read_settings(spider)
if self.enabled:
logger.info("Using Crawlera Fetch API at %s with apikey %s***" %
(self.url, self.apikey[:5]))
def test_getpriority(self):
settings = BaseSettings({'key': 'value'}, priority=99)
self.assertEqual(settings.getpriority('key'), 99)
self.assertEqual(settings.getpriority('nonexistentkey'), None)
def setUp(self):
self.settings = BaseSettings()
def test_update(self):
settings = BaseSettings({'key_lowprio': 0}, priority=0)
settings.set('key_highprio', 10, priority=50)
custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11}, priority=30)
custom_settings.set('newkey_one', None, priority=50)
custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None}
settings.update(custom_dict, priority=20)
self.assertEqual(settings['key_lowprio'], 2)
self.assertEqual(settings.getpriority('key_lowprio'), 20)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_two', settings)
self.assertEqual(settings.getpriority('newkey_two'), 20)
settings.update(custom_settings)
self.assertEqual(settings['key_lowprio'], 1)
self.assertEqual(settings.getpriority('key_lowprio'), 30)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_one', settings)
self.assertEqual(settings.getpriority('newkey_one'), 50)
settings.update({'key_lowprio': 3}, priority=20)
self.assertEqual(settings['key_lowprio'], 1)
def setUp(self):
self.settings = BaseSettings()
class BaseSettingsTest(unittest.TestCase):
if six.PY3:
assertItemsEqual = unittest.TestCase.assertCountEqual
def setUp(self):
self.settings = BaseSettings()
def test_set_new_attribute(self):
self.settings.set('TEST_OPTION', 'value', 0)
self.assertIn('TEST_OPTION', self.settings.attributes)
attr = self.settings.attributes['TEST_OPTION']
self.assertIsInstance(attr, SettingsAttribute)
self.assertEqual(attr.value, 'value')
self.assertEqual(attr.priority, 0)
def test_set_settingsattribute(self):
myattr = SettingsAttribute(0, 30) # Note priority 30
self.settings.set('TEST_ATTR', myattr, 10)
self.assertEqual(self.settings.get('TEST_ATTR'), 0)
self.assertEqual(self.settings.getpriority('TEST_ATTR'), 30)
def test_set_instance_identity_on_update(self):
attr = SettingsAttribute('value', 0)
self.settings.attributes = {'TEST_OPTION': attr}
self.settings.set('TEST_OPTION', 'othervalue', 10)
self.assertIn('TEST_OPTION', self.settings.attributes)
self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
def test_set_calls_settings_attributes_methods_on_update(self):
attr = SettingsAttribute('value', 10)
with mock.patch.object(attr, '__setattr__') as mock_setattr, \
mock.patch.object(attr, 'set') as mock_set:
self.settings.attributes = {'TEST_OPTION': attr}
for priority in (0, 10, 20):
self.settings.set('TEST_OPTION', 'othervalue', priority)
mock_set.assert_called_once_with('othervalue', priority)
self.assertFalse(mock_setattr.called)
mock_set.reset_mock()
mock_setattr.reset_mock()
def test_setitem(self):
settings = BaseSettings()
settings.set('key', 'a', 'default')
settings['key'] = 'b'
self.assertEqual(settings['key'], 'b')
self.assertEqual(settings.getpriority('key'), 20)
settings['key'] = 'c'
self.assertEqual(settings['key'], 'c')
settings['key2'] = 'x'
self.assertIn('key2', settings)
self.assertEqual(settings['key2'], 'x')
self.assertEqual(settings.getpriority('key2'), 20)
def test_setdict_alias(self):
with mock.patch.object(self.settings, 'set') as mock_set:
self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
self.assertEqual(mock_set.call_count, 2)
calls = [
mock.call('TEST_1', 'value1', 10),
mock.call('TEST_2', 'value2', 10)
]
mock_set.assert_has_calls(calls, any_order=True)
def test_setmodule_only_load_uppercase_vars(self):
class ModuleMock():
UPPERCASE_VAR = 'value'
MIXEDcase_VAR = 'othervalue'
lowercase_var = 'anothervalue'
self.settings.attributes = {}
self.settings.setmodule(ModuleMock(), 10)
self.assertIn('UPPERCASE_VAR', self.settings.attributes)
self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
self.assertNotIn('lowercase_var', self.settings.attributes)
self.assertEqual(len(self.settings.attributes), 1)
def test_setmodule_alias(self):
with mock.patch.object(self.settings, 'set') as mock_set:
self.settings.setmodule(default_settings, 10)
mock_set.assert_any_call('TEST_DEFAULT', 'defvalue', 10)
mock_set.assert_any_call('TEST_DICT', {'key': 'val'}, 10)
def test_setmodule_by_path(self):
self.settings.attributes = {}
self.settings.setmodule(default_settings, 10)
ctrl_attributes = self.settings.attributes.copy()
self.settings.attributes = {}
self.settings.setmodule('tests.test_settings.default_settings', 10)
self.assertItemsEqual(six.iterkeys(self.settings.attributes),
six.iterkeys(ctrl_attributes))
for key in six.iterkeys(ctrl_attributes):
attr = self.settings.attributes[key]
ctrl_attr = ctrl_attributes[key]
self.assertEqual(attr.value, ctrl_attr.value)
self.assertEqual(attr.priority, ctrl_attr.priority)
def test_update(self):
settings = BaseSettings({'key_lowprio': 0}, priority=0)
settings.set('key_highprio', 10, priority=50)
custom_settings = BaseSettings({
'key_lowprio': 1,
'key_highprio': 11
},
priority=30)
custom_settings.set('newkey_one', None, priority=50)
custom_dict = {
'key_lowprio': 2,
'key_highprio': 12,
'newkey_two': None
}
settings.update(custom_dict, priority=20)
self.assertEqual(settings['key_lowprio'], 2)
self.assertEqual(settings.getpriority('key_lowprio'), 20)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_two', settings)
self.assertEqual(settings.getpriority('newkey_two'), 20)
settings.update(custom_settings)
self.assertEqual(settings['key_lowprio'], 1)
self.assertEqual(settings.getpriority('key_lowprio'), 30)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_one', settings)
self.assertEqual(settings.getpriority('newkey_one'), 50)
settings.update({'key_lowprio': 3}, priority=20)
self.assertEqual(settings['key_lowprio'], 1)
def test_update_jsonstring(self):
settings = BaseSettings({
'number': 0,
'dict': BaseSettings({'key': 'val'})
})
settings.update('{"number": 1, "newnumber": 2}')
self.assertEqual(settings['number'], 1)
self.assertEqual(settings['newnumber'], 2)
settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
self.assertEqual(settings['dict']['key'], "newval")
self.assertEqual(settings['dict']['newkey'], "newval2")
def test_delete(self):
settings = BaseSettings({'key': None})
settings.set('key_highprio', None, priority=50)
settings.delete('key')
settings.delete('key_highprio')
self.assertNotIn('key', settings)
self.assertIn('key_highprio', settings)
del settings['key_highprio']
self.assertNotIn('key_highprio', settings)
def test_get(self):
test_configuration = {
'TEST_ENABLED1': '1',
'TEST_ENABLED2': True,
'TEST_ENABLED3': 1,
'TEST_DISABLED1': '0',
'TEST_DISABLED2': False,
'TEST_DISABLED3': 0,
'TEST_INT1': 123,
'TEST_INT2': '123',
'TEST_FLOAT1': 123.45,
'TEST_FLOAT2': '123.45',
'TEST_LIST1': ['one', 'two'],
'TEST_LIST2': 'one,two',
'TEST_STR': 'value',
'TEST_DICT1': {
'key1': 'val1',
'ke2': 3
},
'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
}
settings = self.settings
settings.attributes = {
key: SettingsAttribute(value, 0)
for key, value in six.iteritems(test_configuration)
}
self.assertTrue(settings.getbool('TEST_ENABLED1'))
self.assertTrue(settings.getbool('TEST_ENABLED2'))
self.assertTrue(settings.getbool('TEST_ENABLED3'))
self.assertFalse(settings.getbool('TEST_ENABLEDx'))
self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
self.assertFalse(settings.getbool('TEST_DISABLED1'))
self.assertFalse(settings.getbool('TEST_DISABLED2'))
self.assertFalse(settings.getbool('TEST_DISABLED3'))
self.assertEqual(settings.getint('TEST_INT1'), 123)
self.assertEqual(settings.getint('TEST_INT2'), 123)
self.assertEqual(settings.getint('TEST_INTx'), 0)
self.assertEqual(settings.getint('TEST_INTx', 45), 45)
self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
self.assertEqual(settings.getlist('TEST_LISTx'), [])
self.assertEqual(settings.getlist('TEST_LISTx', ['default']),
['default'])
self.assertEqual(settings['TEST_STR'], 'value')
self.assertEqual(settings.get('TEST_STR'), 'value')
self.assertEqual(settings['TEST_STRx'], None)
self.assertEqual(settings.get('TEST_STRx'), None)
self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
self.assertEqual(settings.getdict('TEST_DICT1'), {
'key1': 'val1',
'ke2': 3
})
self.assertEqual(settings.getdict('TEST_DICT2'), {
'key1': 'val1',
'ke2': 3
})
self.assertEqual(settings.getdict('TEST_DICT3'), {})
self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}),
{'key1': 5})
self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
def test_getpriority(self):
settings = BaseSettings({'key': 'value'}, priority=99)
self.assertEqual(settings.getpriority('key'), 99)
self.assertEqual(settings.getpriority('nonexistentkey'), None)
def test_getcomposite(self):
s = BaseSettings({
'TEST_BASE': {
1: 1,
2: 2
},
'TEST': BaseSettings({
1: 10,
3: 30
}, 'default'),
'HASNOBASE': BaseSettings({1: 1}, 'default')
})
s['TEST'].set(4, 4, priority='project')
# When users specify a _BASE setting they explicitly don't want to use
# Scrapy's defaults, so we don't want to see anything that has a
# 'default' priority from TEST
cs = s._getcomposite('TEST')
self.assertEqual(len(cs), 3)
self.assertEqual(cs[1], 1)
self.assertEqual(cs[2], 2)
self.assertEqual(cs[4], 4)
cs = s._getcomposite('HASNOBASE')
self.assertEqual(len(cs), 1)
self.assertEqual(cs[1], 1)
cs = s._getcomposite('NONEXISTENT')
self.assertIsNone(cs)
def test_maxpriority(self):
# Empty settings should return 'default'
self.assertEqual(self.settings.maxpriority(), 0)
self.settings.set('A', 0, 10)
self.settings.set('B', 0, 30)
self.assertEqual(self.settings.maxpriority(), 30)
def test_copy(self):
values = {
'TEST_BOOL':
True,
'TEST_LIST': ['one', 'two'],
'TEST_LIST_OF_LISTS': [['first_one', 'first_two'],
['second_one', 'second_two']]
}
self.settings.setdict(values)
copy = self.settings.copy()
self.settings.set('TEST_BOOL', False)
self.assertTrue(copy.get('TEST_BOOL'))
test_list = self.settings.get('TEST_LIST')
test_list.append('three')
self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two'])
test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS')
test_list_of_lists[0].append('first_three')
self.assertListEqual(
copy.get('TEST_LIST_OF_LISTS')[0], ['first_one', 'first_two'])
def test_freeze(self):
self.settings.freeze()
with self.assertRaises(TypeError) as cm:
self.settings.set('TEST_BOOL', False)
self.assertEqual(str(cm.exception),
"Trying to modify an immutable Settings object")
def test_frozencopy(self):
frozencopy = self.settings.frozencopy()
self.assertTrue(frozencopy.frozen)
self.assertIsNot(frozencopy, self.settings)
def test_deprecated_attribute_overrides(self):
self.settings.set('BAR', 'fuz', priority='cmdline')
with warnings.catch_warnings(record=True) as w:
self.settings.overrides['BAR'] = 'foo'
self.assertIn("Settings.overrides", str(w[0].message))
self.assertEqual(self.settings.get('BAR'), 'foo')
self.assertEqual(self.settings.overrides.get('BAR'), 'foo')
self.assertIn('BAR', self.settings.overrides)
self.settings.overrides.update(BAR='bus')
self.assertEqual(self.settings.get('BAR'), 'bus')
self.assertEqual(self.settings.overrides.get('BAR'), 'bus')
self.settings.overrides.setdefault('BAR', 'fez')
self.assertEqual(self.settings.get('BAR'), 'bus')
self.settings.overrides.setdefault('FOO', 'fez')
self.assertEqual(self.settings.get('FOO'), 'fez')
self.assertEqual(self.settings.overrides.get('FOO'), 'fez')
def test_deprecated_attribute_defaults(self):
self.settings.set('BAR', 'fuz', priority='default')
with warnings.catch_warnings(record=True) as w:
self.settings.defaults['BAR'] = 'foo'
self.assertIn("Settings.defaults", str(w[0].message))
self.assertEqual(self.settings.get('BAR'), 'foo')
self.assertEqual(self.settings.defaults.get('BAR'), 'foo')
self.assertIn('BAR', self.settings.defaults)
def test_getpriority(self):
settings = BaseSettings({'key': 'value'}, priority=99)
self.assertEqual(settings.getpriority('key'), 99)
self.assertEqual(settings.getpriority('nonexistentkey'), None)
class BaseSettingsTest(unittest.TestCase):
if six.PY3:
assertItemsEqual = unittest.TestCase.assertCountEqual
def setUp(self):
self.settings = BaseSettings()
def test_set_new_attribute(self):
self.settings.set('TEST_OPTION', 'value', 0)
self.assertIn('TEST_OPTION', self.settings.attributes)
attr = self.settings.attributes['TEST_OPTION']
self.assertIsInstance(attr, SettingsAttribute)
self.assertEqual(attr.value, 'value')
self.assertEqual(attr.priority, 0)
def test_set_settingsattribute(self):
myattr = SettingsAttribute(0, 30) # Note priority 30
self.settings.set('TEST_ATTR', myattr, 10)
self.assertEqual(self.settings.get('TEST_ATTR'), 0)
self.assertEqual(self.settings.getpriority('TEST_ATTR'), 30)
def test_set_instance_identity_on_update(self):
attr = SettingsAttribute('value', 0)
self.settings.attributes = {'TEST_OPTION': attr}
self.settings.set('TEST_OPTION', 'othervalue', 10)
self.assertIn('TEST_OPTION', self.settings.attributes)
self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
def test_set_calls_settings_attributes_methods_on_update(self):
attr = SettingsAttribute('value', 10)
with mock.patch.object(attr, '__setattr__') as mock_setattr, \
mock.patch.object(attr, 'set') as mock_set:
self.settings.attributes = {'TEST_OPTION': attr}
for priority in (0, 10, 20):
self.settings.set('TEST_OPTION', 'othervalue', priority)
mock_set.assert_called_once_with('othervalue', priority)
self.assertFalse(mock_setattr.called)
mock_set.reset_mock()
mock_setattr.reset_mock()
def test_setitem(self):
settings = BaseSettings()
settings.set('key', 'a', 'default')
settings['key'] = 'b'
self.assertEqual(settings['key'], 'b')
self.assertEqual(settings.getpriority('key'), 20)
settings['key'] = 'c'
self.assertEqual(settings['key'], 'c')
settings['key2'] = 'x'
self.assertIn('key2', settings)
self.assertEqual(settings['key2'], 'x')
self.assertEqual(settings.getpriority('key2'), 20)
def test_setdict_alias(self):
with mock.patch.object(self.settings, 'set') as mock_set:
self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
self.assertEqual(mock_set.call_count, 2)
calls = [mock.call('TEST_1', 'value1', 10),
mock.call('TEST_2', 'value2', 10)]
mock_set.assert_has_calls(calls, any_order=True)
def test_setmodule_only_load_uppercase_vars(self):
class ModuleMock():
UPPERCASE_VAR = 'value'
MIXEDcase_VAR = 'othervalue'
lowercase_var = 'anothervalue'
self.settings.attributes = {}
self.settings.setmodule(ModuleMock(), 10)
self.assertIn('UPPERCASE_VAR', self.settings.attributes)
self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
self.assertNotIn('lowercase_var', self.settings.attributes)
self.assertEqual(len(self.settings.attributes), 1)
def test_setmodule_alias(self):
with mock.patch.object(self.settings, 'set') as mock_set:
self.settings.setmodule(default_settings, 10)
mock_set.assert_any_call('TEST_DEFAULT', 'defvalue', 10)
mock_set.assert_any_call('TEST_DICT', {'key': 'val'}, 10)
def test_setmodule_by_path(self):
self.settings.attributes = {}
self.settings.setmodule(default_settings, 10)
ctrl_attributes = self.settings.attributes.copy()
self.settings.attributes = {}
self.settings.setmodule(
'tests.test_settings.default_settings', 10)
self.assertItemsEqual(six.iterkeys(self.settings.attributes),
six.iterkeys(ctrl_attributes))
for key in six.iterkeys(ctrl_attributes):
attr = self.settings.attributes[key]
ctrl_attr = ctrl_attributes[key]
self.assertEqual(attr.value, ctrl_attr.value)
self.assertEqual(attr.priority, ctrl_attr.priority)
def test_update(self):
settings = BaseSettings({'key_lowprio': 0}, priority=0)
settings.set('key_highprio', 10, priority=50)
custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11}, priority=30)
custom_settings.set('newkey_one', None, priority=50)
custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None}
settings.update(custom_dict, priority=20)
self.assertEqual(settings['key_lowprio'], 2)
self.assertEqual(settings.getpriority('key_lowprio'), 20)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_two', settings)
self.assertEqual(settings.getpriority('newkey_two'), 20)
settings.update(custom_settings)
self.assertEqual(settings['key_lowprio'], 1)
self.assertEqual(settings.getpriority('key_lowprio'), 30)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_one', settings)
self.assertEqual(settings.getpriority('newkey_one'), 50)
settings.update({'key_lowprio': 3}, priority=20)
self.assertEqual(settings['key_lowprio'], 1)
def test_update_jsonstring(self):
settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
settings.update('{"number": 1, "newnumber": 2}')
self.assertEqual(settings['number'], 1)
self.assertEqual(settings['newnumber'], 2)
settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
self.assertEqual(settings['dict']['key'], "newval")
self.assertEqual(settings['dict']['newkey'], "newval2")
def test_delete(self):
settings = BaseSettings({'key': None})
settings.set('key_highprio', None, priority=50)
settings.delete('key')
settings.delete('key_highprio')
self.assertNotIn('key', settings)
self.assertIn('key_highprio', settings)
del settings['key_highprio']
self.assertNotIn('key_highprio', settings)
def test_get(self):
test_configuration = {
'TEST_ENABLED1': '1',
'TEST_ENABLED2': True,
'TEST_ENABLED3': 1,
'TEST_DISABLED1': '0',
'TEST_DISABLED2': False,
'TEST_DISABLED3': 0,
'TEST_INT1': 123,
'TEST_INT2': '123',
'TEST_FLOAT1': 123.45,
'TEST_FLOAT2': '123.45',
'TEST_LIST1': ['one', 'two'],
'TEST_LIST2': 'one,two',
'TEST_STR': 'value',
'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
}
settings = self.settings
settings.attributes = {key: SettingsAttribute(value, 0) for key, value
in six.iteritems(test_configuration)}
self.assertTrue(settings.getbool('TEST_ENABLED1'))
self.assertTrue(settings.getbool('TEST_ENABLED2'))
self.assertTrue(settings.getbool('TEST_ENABLED3'))
self.assertFalse(settings.getbool('TEST_ENABLEDx'))
self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
self.assertFalse(settings.getbool('TEST_DISABLED1'))
self.assertFalse(settings.getbool('TEST_DISABLED2'))
self.assertFalse(settings.getbool('TEST_DISABLED3'))
self.assertEqual(settings.getint('TEST_INT1'), 123)
self.assertEqual(settings.getint('TEST_INT2'), 123)
self.assertEqual(settings.getint('TEST_INTx'), 0)
self.assertEqual(settings.getint('TEST_INTx', 45), 45)
self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
self.assertEqual(settings.getlist('TEST_LISTx'), [])
self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
self.assertEqual(settings['TEST_STR'], 'value')
self.assertEqual(settings.get('TEST_STR'), 'value')
self.assertEqual(settings['TEST_STRx'], None)
self.assertEqual(settings.get('TEST_STRx'), None)
self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
self.assertEqual(settings.getdict('TEST_DICT3'), {})
self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
def test_getpriority(self):
settings = BaseSettings({'key': 'value'}, priority=99)
self.assertEqual(settings.getpriority('key'), 99)
self.assertEqual(settings.getpriority('nonexistentkey'), None)
def test_getcomposite(self):
s = BaseSettings({'TEST_BASE': {1: 1, 2: 2},
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({1: 1}, 'default')})
s['TEST'].set(4, 4, priority='project')
# When users specify a _BASE setting they explicitly don't want to use
# Scrapy's defaults, so we don't want to see anything that has a
# 'default' priority from TEST
cs = s._getcomposite('TEST')
self.assertEqual(len(cs), 3)
self.assertEqual(cs[1], 1)
self.assertEqual(cs[2], 2)
self.assertEqual(cs[4], 4)
cs = s._getcomposite('HASNOBASE')
self.assertEqual(len(cs), 1)
self.assertEqual(cs[1], 1)
cs = s._getcomposite('NONEXISTENT')
self.assertIsNone(cs)
def test_maxpriority(self):
# Empty settings should return 'default'
self.assertEqual(self.settings.maxpriority(), 0)
self.settings.set('A', 0, 10)
self.settings.set('B', 0, 30)
self.assertEqual(self.settings.maxpriority(), 30)
def test_copy(self):
values = {
'TEST_BOOL': True,
'TEST_LIST': ['one', 'two'],
'TEST_LIST_OF_LISTS': [['first_one', 'first_two'],
['second_one', 'second_two']]
}
self.settings.setdict(values)
copy = self.settings.copy()
self.settings.set('TEST_BOOL', False)
self.assertTrue(copy.get('TEST_BOOL'))
test_list = self.settings.get('TEST_LIST')
test_list.append('three')
self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two'])
test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS')
test_list_of_lists[0].append('first_three')
self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0],
['first_one', 'first_two'])
def test_freeze(self):
self.settings.freeze()
with self.assertRaises(TypeError) as cm:
self.settings.set('TEST_BOOL', False)
self.assertEqual(str(cm.exception),
"Trying to modify an immutable Settings object")
def test_frozencopy(self):
frozencopy = self.settings.frozencopy()
self.assertTrue(frozencopy.frozen)
self.assertIsNot(frozencopy, self.settings)
def test_deprecated_attribute_overrides(self):
self.settings.set('BAR', 'fuz', priority='cmdline')
with warnings.catch_warnings(record=True) as w:
self.settings.overrides['BAR'] = 'foo'
self.assertIn("Settings.overrides", str(w[0].message))
self.assertEqual(self.settings.get('BAR'), 'foo')
self.assertEqual(self.settings.overrides.get('BAR'), 'foo')
self.assertIn('BAR', self.settings.overrides)
self.settings.overrides.update(BAR='bus')
self.assertEqual(self.settings.get('BAR'), 'bus')
self.assertEqual(self.settings.overrides.get('BAR'), 'bus')
self.settings.overrides.setdefault('BAR', 'fez')
self.assertEqual(self.settings.get('BAR'), 'bus')
self.settings.overrides.setdefault('FOO', 'fez')
self.assertEqual(self.settings.get('FOO'), 'fez')
self.assertEqual(self.settings.overrides.get('FOO'), 'fez')
def test_deprecated_attribute_defaults(self):
self.settings.set('BAR', 'fuz', priority='default')
with warnings.catch_warnings(record=True) as w:
self.settings.defaults['BAR'] = 'foo'
self.assertIn("Settings.defaults", str(w[0].message))
self.assertEqual(self.settings.get('BAR'), 'foo')
self.assertEqual(self.settings.defaults.get('BAR'), 'foo')
self.assertIn('BAR', self.settings.defaults)
def test_update(self):
settings = BaseSettings({'key_lowprio': 0}, priority=0)
settings.set('key_highprio', 10, priority=50)
custom_settings = BaseSettings({
'key_lowprio': 1,
'key_highprio': 11
},
priority=30)
custom_settings.set('newkey_one', None, priority=50)
custom_dict = {
'key_lowprio': 2,
'key_highprio': 12,
'newkey_two': None
}
settings.update(custom_dict, priority=20)
self.assertEqual(settings['key_lowprio'], 2)
self.assertEqual(settings.getpriority('key_lowprio'), 20)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_two', settings)
self.assertEqual(settings.getpriority('newkey_two'), 20)
settings.update(custom_settings)
self.assertEqual(settings['key_lowprio'], 1)
self.assertEqual(settings.getpriority('key_lowprio'), 30)
self.assertEqual(settings['key_highprio'], 10)
self.assertIn('newkey_one', settings)
self.assertEqual(settings.getpriority('newkey_one'), 50)
settings.update({'key_lowprio': 3}, priority=20)
self.assertEqual(settings['key_lowprio'], 1)
def from_crawler(cls, crawler: Crawler):
base_settings: BaseSettings = crawler.settings
cls.normalize(base_settings)
settings = BaseSettings(priority='spider')
cls.from_object(settings, crawler.spidercls.SpiderConfig)
settings.update(
{k: v
for k, v in base_settings.items() if k in settings},
priority='cmdline')
preset = base_settings.get('PRESET')
if preset:
preset_dict = BaseSettings(priority=35)
cls.from_pyfile(preset_dict, preset)
settings.update(preset_dict)
adapted = BaseSettings(priority=50)
for k, v in settings.items():
adapt = getattr(SettingsAdapter, k.lower(), None)
if adapt:
adapted.update(adapt(v))
else:
adapted[k] = v
base_settings.update(adapted.copy_to_dict(), priority=50)
base_settings['SPIDER_CONFIG'] = adapted
return cls()