def __init__(self, crawler):
self.crawler = crawler
self.settings = crawler.settings
self.feeds = {}
self.slots = []
if not self.settings['FEEDS'] and not self.settings['FEED_URI']:
raise NotConfigured
# Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings
if self.settings['FEED_URI']:
warnings.warn(
'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of '
'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details',
category=ScrapyDeprecationWarning, stacklevel=2,
)
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
# 'FEEDS' setting takes precedence over 'FEED_URI'
for uri, feed_options in self.settings.getdict('FEEDS').items():
uri = str(uri) # handle pathlib.Path objects
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
for uri, feed_options in self.feeds.items():
if not self._storage_supported(uri, feed_options):
raise NotConfigured
if not self._settings_are_valid():
raise NotConfigured
if not self._exporter_supported(feed_options['format']):
raise NotConfigured
def test_feed_complete_default_values_from_settings_empty(self):
feed = {}
settings = Settings({
"FEED_EXPORT_ENCODING": "custom encoding",
"FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
"FEED_EXPORT_INDENT": 42,
"FEED_STORE_EMPTY": True,
"FEED_URI_PARAMS": (1, 2, 3, 4),
})
new_feed = feed_complete_default_values_from_settings(feed, settings)
self.assertEqual(
new_feed, {
"encoding": "custom encoding",
"fields": ["f1", "f2", "f3"],
"indent": 42,
"store_empty": True,
"uri_params": (1, 2, 3, 4),
})
def test_feed_complete_default_values_from_settings_non_empty(self):
feed = {
"encoding": "other encoding",
"fields": None,
}
settings = Settings({
"FEED_EXPORT_ENCODING": "custom encoding",
"FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
"FEED_EXPORT_INDENT": 42,
"FEED_STORE_EMPTY": True,
"FEED_EXPORT_BATCH_ITEM_COUNT": 2,
})
new_feed = feed_complete_default_values_from_settings(feed, settings)
self.assertEqual(new_feed, {
"encoding": "other encoding",
"fields": None,
"indent": 42,
"store_empty": True,
"uri_params": None,
"batch_item_count": 2,
})
