def __init__(self, crawler): self.crawler = crawler self.settings = crawler.settings self.feeds = {} self.slots = [] if not self.settings['FEEDS'] and not self.settings['FEED_URI']: raise NotConfigured # Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings if self.settings['FEED_URI']: warnings.warn( 'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of ' 'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details', category=ScrapyDeprecationWarning, stacklevel=2, ) uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') for uri, feed_options in self.feeds.items(): if not self._storage_supported(uri, feed_options): raise NotConfigured if not self._settings_are_valid(): raise NotConfigured if not self._exporter_supported(feed_options['format']): raise NotConfigured
def test_feed_complete_default_values_from_settings_empty(self): feed = {} settings = Settings({ "FEED_EXPORT_ENCODING": "custom encoding", "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual( new_feed, { "encoding": "custom encoding", "fields": ["f1", "f2", "f3"], "indent": 42, "store_empty": True, "uri_params": (1, 2, 3, 4), })
def test_feed_complete_default_values_from_settings_non_empty(self): feed = { "encoding": "other encoding", "fields": None, } settings = Settings({ "FEED_EXPORT_ENCODING": "custom encoding", "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { "encoding": "other encoding", "fields": None, "indent": 42, "store_empty": True, "uri_params": None, "batch_item_count": 2, })