class Scraper(injection.Component):
config = injection.Infrastructure()
def __init__(self, name, container) -> None:
super(Scraper, self).__init__(name, container)
self.settings = Settings()
self.settings.setdict(self.config)
self.settings.set("CONTAINER", container)
self._process = None
def crawl(self, spider, loglevel, settings_override=None):
if self._process:
self._process.stop()
settings = self.settings.copy()
if settings_override:
settings.update(settings_override, priority="cmdline")
if loglevel:
settings.set("LOG_ENABLED", True, priority="cmdline")
settings.set("LOG_LEVEL", loglevel, priority="cmdline")
self._process = CrawlerProcess(settings)
self._process.crawl(spider)
self._process.start()
def list_spiders(self):
loader = SpiderLoader.from_settings(self.settings)
return loader.list()
def __init__(self, spidercls, settings=None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings=None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings) # 统一
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings: %(settings)r", {'settings': d})
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
# 复位？
self.settings.freeze() # settings不可改变了
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings=None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings):
if isinstance(settings, dict):
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
self.signals.connect(lambda: logging.root.removeHandler(handler),
signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.spidercls.update_settings(self.settings)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings):
if isinstance(settings, dict):
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
self.signals.connect(lambda: logging.root.removeHandler(handler),
signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.spidercls.update_settings(self.settings)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
if isinstance(spidercls, Spider):
raise ValueError(
'The spidercls argument must be a class, not an object')
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings:\n%(settings)s",
{'settings': pprint.pformat(d)})
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance(
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
settings=self.settings,
crawler=self,
)
reactor_class = self.settings.get("TWISTED_REACTOR")
if init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
if reactor_class:
install_reactor(reactor_class,
self.settings["ASYNCIO_EVENT_LOOP"])
else:
from twisted.internet import reactor # noqa: F401
log_reactor_info()
if reactor_class:
verify_installed_reactor(reactor_class)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings=None):
## crawler 对象必须用 scrapy.spiders.Spider 的子类和一个 scrapy.settings.Settings
## 对象来实例化
if isinstance(spidercls, Spider):
raise ValueError(
'The spidercls argument must be a class, not an object')
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
## 自定义爬虫类
self.spidercls = spidercls
## crawler 的配置管理器，用来为插件和中间件提供访问该 crawler 的 Scrapy 配置的入口
self.settings = settings.copy()
## 根据自定义爬虫类中的可能定义的 custom_settigns 属性更新配置
## 优先级为 spider
self.spidercls.update_settings(self.settings)
## 这里得到的只是被覆盖过的配置项，并将其转换为字典
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings: %(settings)r", {'settings': d})
## crawler 的信号管理器，被插件和中间件用来将它们自身集成到 Scrapy 功能中
self.signals = SignalManager(self)
## crawler 的 stats 收集器，用来从插件和中间件中记录它们的行为和访问其他插件收集到的数据
self.stats = load_object(self.settings['STATS_CLASS'])(self)
## 用于对爬虫运行过程中产生的日志的级别数量，进行统计
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
## 为 engine_stopped 信号注册 __remove_handler 处理器
## 当产生引擎停止信号时，将会由 __remove_handler 处理器进行处理
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
## 初始化日志格式化器实例
self.logformatter = lf_cls.from_crawler(self)
## 用来追踪可用插件的插件管理器
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
## 标志爬虫运行状态
self.crawling = False
## 当前正在爬取的 spider
self.spider = None
## 执行引擎，用来协调调度器、下载器、spiders 之间的爬取逻辑
self.engine = None
def qwebkit_settings(settings=None):
if settings is None:
settings = Settings()
elif settings.getbool("__WT__"):
return settings
else:
settings = settings.copy()
settings.frozen = False
for name in dir(defs):
if name.startswith("WT_") and settings.get(name) is None:
settings.set(name, getattr(defs, name))
settings.set("__WT__", True)
return settings
def __init__(self, spidercls, settings=None):
if isinstance(spidercls, Spider):
# spidercls参数必须是类，而不是对象
raise ValueError(
'The spidercls argument must be a class, not an object')
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
# 加载配置文件
self.settings = settings.copy()
# 这里调用了更新设置
self.spidercls.update_settings(self.settings)
# 初始化信号管理类
self.signals = SignalManager(self)
# 初始化日志收集类
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings:\n%(settings)s",
{'settings': pprint.pformat(d)})
# 这个是关于根日志的
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
# 加lambda为了防止垃圾回收机制？
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
# 加载日志格式化的东西
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
# 好像是个扩展
self.extensions = ExtensionManager.from_crawler(self)
# settings初始化完成，禁止修改了
self.settings.freeze()
# 未开始抓取
self.crawling = False
# 这里做准备操作，后面crawl才进行赋值
self.spider = None
self.engine = None
def __init__(self, spidercls, settings=None):
if isinstance(settings, dict) or settings is None: # 此处应该直接就是一个False
settings = Settings(settings)
self.spidercls = spidercls # 获取了爬虫对象，，但是还是没有实例化 # 所以流程就是获取爬虫对象，在实例化之前，执行了一次update_settings，就是针对custom setting的一次操作嘛
self.settings = settings.copy()
self.spidercls.update_settings(
self.settings
) # 就是在这里执行了对custom_setting的设置嘛，可以很强，把爬虫里面的custom_setting更新到setting里面?好吧没事共同维护的一个setting对象，确实是直接更新到了setting里面
# 所以在实例化之前，他只是更新了settings而已，你写在__init__其实一点用都得的，根本就没有执行到哪一步
# todo 总结。我以前是想把custiom_settings卸载init里面，但是并没有起作用，因为此时的爬虫也就是 self.spidercls 根本就还没有进行初始化，而是先执行的update_settings
d = dict(overridden_settings(self.settings)) # 找出相同的，取overridden里面的值
logger.info(
"Overridden settings: %(settings)r",
{'settings': d
}) # 这就是打印那句log的地方，打印出Overridden属性。现遍历默认属性，找出已有属性中对应的值，看那些有修改
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(
self
) # STATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector' -- 统计机制
handler = LogCounterHandler(
self, level=self.settings.get('LOG_LEVEL')) # LOG_LEVEL = 'DEBUG'
logging.root.addHandler(handler)
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(
self.settings['LOG_FORMATTER']
) # LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(
self) # extensions是干嘛用的，杵这干啥呢，闹呢。卧槽，貌似不需要使用这些扩展件呀
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings):
if isinstance(settings, dict):
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.spidercls.update_settings(self.settings)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings):
if isinstance(settings, dict):
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.spidercls.update_settings(self.settings)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def get_crawler_process(name):
'''
建立爬虫进程，开始爬取
'''
custom_settings = get_config(name)
'''
获取爬虫配置信息
'''
spider = custom_settings.get('spider', 'universal')
project_settings = Settings()
project_settings.setmodule("universal.settings")
settings = dict(project_settings.copy())
'''
根据爬虫配置信息中的settings属性，更新settings.py相应配置项
'''
settings.update(custom_settings.get('settings', {}))
#开启爬虫进程
process = CrawlerProcess(settings)
process.crawl(spider, **{'name': name})
return process
def __init__(self, spidercls, settings=None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings: %(settings)r", {'settings': d})
"""
通信机制:
SignalManager主要是利用开源的python库pydispatch作消息的发送和路由.
scrapy使用它发送关键的消息事件给关心者，如爬取开始，爬取结束等消息
通过send_catch_log_deferred来发送消息，通过connect方法来注册关心消息的处理函数
"""
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings=None):
if isinstance(spidercls, Spider):
raise ValueError(
"The spidercls argument must be a class, not an object")
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
self.stats = load_object(self.settings["STATS_CLASS"])(self)
handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
logging.root.addHandler(handler)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings:\n%(settings)s",
{"settings": pprint.pformat(d)})
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings["LOG_FORMATTER"])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def __init__(self, spidercls, settings=None):
if isinstance(spidercls, Spider):
raise ValueError(
'The spidercls argument must be a class, not an object')
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings: %(settings)r", {'settings': d})
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
class SettingsTest(unittest.TestCase):
if six.PY3:
assertItemsEqual = unittest.TestCase.assertCountEqual
def setUp(self):
self.settings = Settings()
@mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
@mock.patch('scrapy.settings.default_settings', default_settings)
def test_initial_defaults(self):
settings = Settings()
self.assertEqual(len(settings.attributes), 1)
self.assertIn('TEST_DEFAULT', settings.attributes)
attr = settings.attributes['TEST_DEFAULT']
self.assertIsInstance(attr, SettingsAttribute)
self.assertEqual(attr.value, 'defvalue')
self.assertEqual(attr.priority, 10)
@mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
@mock.patch('scrapy.settings.default_settings', {})
def test_initial_values(self):
settings = Settings({'TEST_OPTION': 'value'}, 10)
self.assertEqual(len(settings.attributes), 1)
self.assertIn('TEST_OPTION', settings.attributes)
attr = settings.attributes['TEST_OPTION']
self.assertIsInstance(attr, SettingsAttribute)
self.assertEqual(attr.value, 'value')
self.assertEqual(attr.priority, 10)
def test_set_new_attribute(self):
self.settings.attributes = {}
self.settings.set('TEST_OPTION', 'value', 0)
self.assertIn('TEST_OPTION', self.settings.attributes)
attr = self.settings.attributes['TEST_OPTION']
self.assertIsInstance(attr, SettingsAttribute)
self.assertEqual(attr.value, 'value')
self.assertEqual(attr.priority, 0)
def test_set_instance_identity_on_update(self):
attr = SettingsAttribute('value', 0)
self.settings.attributes = {'TEST_OPTION': attr}
self.settings.set('TEST_OPTION', 'othervalue', 10)
self.assertIn('TEST_OPTION', self.settings.attributes)
self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
def test_set_calls_settings_attributes_methods_on_update(self):
with mock.patch.object(SettingsAttribute, '__setattr__') as mock_setattr, \
mock.patch.object(SettingsAttribute, 'set') as mock_set:
attr = SettingsAttribute('value', 10)
self.settings.attributes = {'TEST_OPTION': attr}
mock_set.reset_mock()
mock_setattr.reset_mock()
for priority in (0, 10, 20):
self.settings.set('TEST_OPTION', 'othervalue', priority)
mock_set.assert_called_once_with('othervalue', priority)
self.assertFalse(mock_setattr.called)
mock_set.reset_mock()
mock_setattr.reset_mock()
def test_setdict_alias(self):
with mock.patch.object(self.settings, 'set') as mock_set:
self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
self.assertEqual(mock_set.call_count, 2)
calls = [
mock.call('TEST_1', 'value1', 10),
mock.call('TEST_2', 'value2', 10)
]
mock_set.assert_has_calls(calls, any_order=True)
def test_setmodule_only_load_uppercase_vars(self):
class ModuleMock():
UPPERCASE_VAR = 'value'
MIXEDcase_VAR = 'othervalue'
lowercase_var = 'anothervalue'
self.settings.attributes = {}
self.settings.setmodule(ModuleMock(), 10)
self.assertIn('UPPERCASE_VAR', self.settings.attributes)
self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
self.assertNotIn('lowercase_var', self.settings.attributes)
self.assertEqual(len(self.settings.attributes), 1)
def test_setmodule_alias(self):
with mock.patch.object(self.settings, 'set') as mock_set:
self.settings.setmodule(default_settings, 10)
mock_set.assert_called_with('TEST_DEFAULT', 'defvalue', 10)
def test_setmodule_by_path(self):
self.settings.attributes = {}
self.settings.setmodule(default_settings, 10)
ctrl_attributes = self.settings.attributes.copy()
self.settings.attributes = {}
self.settings.setmodule('tests.test_settings.default_settings', 10)
self.assertItemsEqual(six.iterkeys(self.settings.attributes),
six.iterkeys(ctrl_attributes))
for attr, ctrl_attr in zip(six.itervalues(self.settings.attributes),
six.itervalues(ctrl_attributes)):
self.assertEqual(attr.value, ctrl_attr.value)
self.assertEqual(attr.priority, ctrl_attr.priority)
def test_get(self):
test_configuration = {
'TEST_ENABLED1': '1',
'TEST_ENABLED2': True,
'TEST_ENABLED3': 1,
'TEST_DISABLED1': '0',
'TEST_DISABLED2': False,
'TEST_DISABLED3': 0,
'TEST_INT1': 123,
'TEST_INT2': '123',
'TEST_FLOAT1': 123.45,
'TEST_FLOAT2': '123.45',
'TEST_LIST1': ['one', 'two'],
'TEST_LIST2': 'one,two',
'TEST_STR': 'value',
'TEST_DICT1': {
'key1': 'val1',
'ke2': 3
},
'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
}
settings = self.settings
settings.attributes = {
key: SettingsAttribute(value, 0)
for key, value in six.iteritems(test_configuration)
}
self.assertTrue(settings.getbool('TEST_ENABLED1'))
self.assertTrue(settings.getbool('TEST_ENABLED2'))
self.assertTrue(settings.getbool('TEST_ENABLED3'))
self.assertFalse(settings.getbool('TEST_ENABLEDx'))
self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
self.assertFalse(settings.getbool('TEST_DISABLED1'))
self.assertFalse(settings.getbool('TEST_DISABLED2'))
self.assertFalse(settings.getbool('TEST_DISABLED3'))
self.assertEqual(settings.getint('TEST_INT1'), 123)
self.assertEqual(settings.getint('TEST_INT2'), 123)
self.assertEqual(settings.getint('TEST_INTx'), 0)
self.assertEqual(settings.getint('TEST_INTx', 45), 45)
self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
self.assertEqual(settings.getlist('TEST_LISTx'), [])
self.assertEqual(settings.getlist('TEST_LISTx', ['default']),
['default'])
self.assertEqual(settings['TEST_STR'], 'value')
self.assertEqual(settings.get('TEST_STR'), 'value')
self.assertEqual(settings['TEST_STRx'], None)
self.assertEqual(settings.get('TEST_STRx'), None)
self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
self.assertEqual(settings.getdict('TEST_DICT1'), {
'key1': 'val1',
'ke2': 3
})
self.assertEqual(settings.getdict('TEST_DICT2'), {
'key1': 'val1',
'ke2': 3
})
self.assertEqual(settings.getdict('TEST_DICT3'), {})
self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}),
{'key1': 5})
self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
def test_copy(self):
values = {
'TEST_BOOL':
True,
'TEST_LIST': ['one', 'two'],
'TEST_LIST_OF_LISTS': [['first_one', 'first_two'],
['second_one', 'second_two']]
}
self.settings.setdict(values)
copy = self.settings.copy()
self.settings.set('TEST_BOOL', False)
self.assertTrue(copy.get('TEST_BOOL'))
test_list = self.settings.get('TEST_LIST')
test_list.append('three')
self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two'])
test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS')
test_list_of_lists[0].append('first_three')
self.assertListEqual(
copy.get('TEST_LIST_OF_LISTS')[0], ['first_one', 'first_two'])
def test_freeze(self):
self.settings.freeze()
with self.assertRaises(TypeError) as cm:
self.settings.set('TEST_BOOL', False)
self.assertEqual(str(cm.exception),
"Trying to modify an immutable Settings object")
def test_frozencopy(self):
frozencopy = self.settings.frozencopy()
self.assertTrue(frozencopy.frozen)
self.assertIsNot(frozencopy, self.settings)
def test_deprecated_attribute_overrides(self):
self.settings.set('BAR', 'fuz', priority='cmdline')
with warnings.catch_warnings(record=True) as w:
self.settings.overrides['BAR'] = 'foo'
self.assertIn("Settings.overrides", str(w[0].message))
self.assertEqual(self.settings.get('BAR'), 'foo')
self.assertEqual(self.settings.overrides.get('BAR'), 'foo')
self.assertIn('BAR', self.settings.overrides)
self.settings.overrides.update(BAR='bus')
self.assertEqual(self.settings.get('BAR'), 'bus')
self.assertEqual(self.settings.overrides.get('BAR'), 'bus')
self.settings.overrides.setdefault('BAR', 'fez')
self.assertEqual(self.settings.get('BAR'), 'bus')
self.settings.overrides.setdefault('FOO', 'fez')
self.assertEqual(self.settings.get('FOO'), 'fez')
self.assertEqual(self.settings.overrides.get('FOO'), 'fez')
def test_deprecated_attribute_defaults(self):
self.settings.set('BAR', 'fuz', priority='default')
with warnings.catch_warnings(record=True) as w:
self.settings.defaults['BAR'] = 'foo'
self.assertIn("Settings.defaults", str(w[0].message))
self.assertEqual(self.settings.get('BAR'), 'foo')
self.assertEqual(self.settings.defaults.get('BAR'), 'foo')
self.assertIn('BAR', self.settings.defaults)
class SettingsTest(unittest.TestCase):
if six.PY3:
assertItemsEqual = unittest.TestCase.assertCountEqual
def setUp(self):
self.settings = Settings()
@mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
@mock.patch('scrapy.settings.default_settings', default_settings)
def test_initial_defaults(self):
settings = Settings()
self.assertEqual(len(settings.attributes), 1)
self.assertIn('TEST_DEFAULT', settings.attributes)
attr = settings.attributes['TEST_DEFAULT']
self.assertIsInstance(attr, SettingsAttribute)
self.assertEqual(attr.value, 'defvalue')
self.assertEqual(attr.priority, 10)
@mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
@mock.patch('scrapy.settings.default_settings', {})
def test_initial_values(self):
settings = Settings({'TEST_OPTION': 'value'}, 10)
self.assertEqual(len(settings.attributes), 1)
self.assertIn('TEST_OPTION', settings.attributes)
attr = settings.attributes['TEST_OPTION']
self.assertIsInstance(attr, SettingsAttribute)
self.assertEqual(attr.value, 'value')
self.assertEqual(attr.priority, 10)
def test_set_new_attribute(self):
self.settings.attributes = {}
self.settings.set('TEST_OPTION', 'value', 0)
self.assertIn('TEST_OPTION', self.settings.attributes)
attr = self.settings.attributes['TEST_OPTION']
self.assertIsInstance(attr, SettingsAttribute)
self.assertEqual(attr.value, 'value')
self.assertEqual(attr.priority, 0)
def test_set_instance_identity_on_update(self):
attr = SettingsAttribute('value', 0)
self.settings.attributes = {'TEST_OPTION': attr}
self.settings.set('TEST_OPTION', 'othervalue', 10)
self.assertIn('TEST_OPTION', self.settings.attributes)
self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
def test_set_calls_settings_attributes_methods_on_update(self):
with mock.patch.object(SettingsAttribute, '__setattr__') as mock_setattr, \
mock.patch.object(SettingsAttribute, 'set') as mock_set:
attr = SettingsAttribute('value', 10)
self.settings.attributes = {'TEST_OPTION': attr}
mock_set.reset_mock()
mock_setattr.reset_mock()
for priority in (0, 10, 20):
self.settings.set('TEST_OPTION', 'othervalue', priority)
mock_set.assert_called_once_with('othervalue', priority)
self.assertFalse(mock_setattr.called)
mock_set.reset_mock()
mock_setattr.reset_mock()
def test_setdict_alias(self):
with mock.patch.object(self.settings, 'set') as mock_set:
self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
self.assertEqual(mock_set.call_count, 2)
calls = [mock.call('TEST_1', 'value1', 10),
mock.call('TEST_2', 'value2', 10)]
mock_set.assert_has_calls(calls, any_order=True)
def test_setmodule_only_load_uppercase_vars(self):
class ModuleMock():
UPPERCASE_VAR = 'value'
MIXEDcase_VAR = 'othervalue'
lowercase_var = 'anothervalue'
self.settings.attributes = {}
self.settings.setmodule(ModuleMock(), 10)
self.assertIn('UPPERCASE_VAR', self.settings.attributes)
self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
self.assertNotIn('lowercase_var', self.settings.attributes)
self.assertEqual(len(self.settings.attributes), 1)
def test_setmodule_alias(self):
with mock.patch.object(self.settings, 'set') as mock_set:
self.settings.setmodule(default_settings, 10)
mock_set.assert_called_with('TEST_DEFAULT', 'defvalue', 10)
def test_setmodule_by_path(self):
self.settings.attributes = {}
self.settings.setmodule(default_settings, 10)
ctrl_attributes = self.settings.attributes.copy()
self.settings.attributes = {}
self.settings.setmodule(
'tests.test_settings.default_settings', 10)
self.assertItemsEqual(six.iterkeys(self.settings.attributes),
six.iterkeys(ctrl_attributes))
for attr, ctrl_attr in zip(six.itervalues(self.settings.attributes),
six.itervalues(ctrl_attributes)):
self.assertEqual(attr.value, ctrl_attr.value)
self.assertEqual(attr.priority, ctrl_attr.priority)
def test_get(self):
test_configuration = {
'TEST_ENABLED1': '1',
'TEST_ENABLED2': True,
'TEST_ENABLED3': 1,
'TEST_DISABLED1': '0',
'TEST_DISABLED2': False,
'TEST_DISABLED3': 0,
'TEST_INT1': 123,
'TEST_INT2': '123',
'TEST_FLOAT1': 123.45,
'TEST_FLOAT2': '123.45',
'TEST_LIST1': ['one', 'two'],
'TEST_LIST2': 'one,two',
'TEST_STR': 'value',
'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
}
settings = self.settings
settings.attributes = {key: SettingsAttribute(value, 0) for key, value
in six.iteritems(test_configuration)}
self.assertTrue(settings.getbool('TEST_ENABLED1'))
self.assertTrue(settings.getbool('TEST_ENABLED2'))
self.assertTrue(settings.getbool('TEST_ENABLED3'))
self.assertFalse(settings.getbool('TEST_ENABLEDx'))
self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
self.assertFalse(settings.getbool('TEST_DISABLED1'))
self.assertFalse(settings.getbool('TEST_DISABLED2'))
self.assertFalse(settings.getbool('TEST_DISABLED3'))
self.assertEqual(settings.getint('TEST_INT1'), 123)
self.assertEqual(settings.getint('TEST_INT2'), 123)
self.assertEqual(settings.getint('TEST_INTx'), 0)
self.assertEqual(settings.getint('TEST_INTx', 45), 45)
self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
self.assertEqual(settings.getlist('TEST_LISTx'), [])
self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
self.assertEqual(settings['TEST_STR'], 'value')
self.assertEqual(settings.get('TEST_STR'), 'value')
self.assertEqual(settings['TEST_STRx'], None)
self.assertEqual(settings.get('TEST_STRx'), None)
self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
self.assertEqual(settings.getdict('TEST_DICT3'), {})
self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
def test_copy(self):
values = {
'TEST_BOOL': True,
'TEST_LIST': ['one', 'two'],
'TEST_LIST_OF_LISTS': [['first_one', 'first_two'],
['second_one', 'second_two']]
}
self.settings.setdict(values)
copy = self.settings.copy()
self.settings.set('TEST_BOOL', False)
self.assertTrue(copy.get('TEST_BOOL'))
test_list = self.settings.get('TEST_LIST')
test_list.append('three')
self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two'])
test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS')
test_list_of_lists[0].append('first_three')
self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0],
['first_one', 'first_two'])
def test_freeze(self):
self.settings.freeze()
with self.assertRaises(TypeError) as cm:
self.settings.set('TEST_BOOL', False)
self.assertEqual(str(cm.exception),
"Trying to modify an immutable Settings object")
def test_frozencopy(self):
with mock.patch.object(self.settings, 'copy') as mock_copy:
with mock.patch.object(mock_copy, 'freeze') as mock_freeze:
mock_object = self.settings.frozencopy()
mock_copy.assert_call_once()
mock_freeze.assert_call_once()
self.assertEqual(mock_object, mock_copy.return_value)
def test_deprecated_attribute_overrides(self):
self.settings.set('BAR', 'fuz', priority='cmdline')
with warnings.catch_warnings(record=True) as w:
self.settings.overrides['BAR'] = 'foo'
self.assertIn("Settings.overrides", str(w[0].message))
self.assertEqual(self.settings.get('BAR'), 'foo')
self.assertEqual(self.settings.overrides.get('BAR'), 'foo')
self.assertIn('BAR', self.settings.overrides)
self.settings.overrides.update(BAR='bus')
self.assertEqual(self.settings.get('BAR'), 'bus')
self.assertEqual(self.settings.overrides.get('BAR'), 'bus')
self.settings.overrides.setdefault('BAR', 'fez')
self.assertEqual(self.settings.get('BAR'), 'bus')
self.settings.overrides.setdefault('FOO', 'fez')
self.assertEqual(self.settings.get('FOO'), 'fez')
self.assertEqual(self.settings.overrides.get('FOO'), 'fez')
def test_deprecated_attribute_defaults(self):
self.settings.set('BAR', 'fuz', priority='default')
with warnings.catch_warnings(record=True) as w:
self.settings.defaults['BAR'] = 'foo'
self.assertIn("Settings.defaults", str(w[0].message))
self.assertEqual(self.settings.get('BAR'), 'foo')
self.assertEqual(self.settings.defaults.get('BAR'), 'foo')
self.assertIn('BAR', self.settings.defaults)