def test_collector(self):
stats = StatsCollector(self.crawler)
self.assertEqual(stats.get_stats(), {})
self.assertEqual(stats.get_value('anything'), None)
self.assertEqual(stats.get_value('anything', 'default'), 'default')
stats.set_value('test', 'value')
self.assertEqual(stats['test'], 'value')
self.assertEqual(stats.get_stats(), {'test': 'value'})
stats['test'] = 'value2'
self.assertIn('test', stats)
self.assertEqual(stats['test'], 'value2')
self.assertEqual(next(iter(stats)), 'test')
self.assertEqual(stats.get_stats(), {'test': 'value2'})
stats['test'] = 'value'
stats.set_value('test2', 23)
self.assertEqual(stats.get_stats(), {'test': 'value', 'test2': 23})
self.assertEqual(stats.get_value('test2'), 23)
stats.inc_value('test2')
self.assertEqual(stats.get_value('test2'), 24)
stats.inc_value('test2', 6)
self.assertEqual(stats.get_value('test2'), 30)
stats.max_value('test2', 6)
self.assertEqual(stats.get_value('test2'), 30)
stats.max_value('test2', 40)
self.assertEqual(stats.get_value('test2'), 40)
stats.max_value('test3', 1)
self.assertEqual(stats.get_value('test3'), 1)
stats.min_value('test2', 60)
self.assertEqual(stats.get_value('test2'), 40)
stats.min_value('test2', 35)
self.assertEqual(stats.get_value('test2'), 35)
stats.min_value('test4', 7)
self.assertEqual(stats.get_value('test4'), 7)
def setUp(self):
crawler = get_crawler(Spider)
self.spider = crawler._create_spider('scrapytest.org')
self.stats = StatsCollector(crawler)
self.stats.open_spider(self.spider)
self.mw = DepthMiddleware(1, self.stats, True)
def setUp(self):
self.spider_name = 'df_tests'
self.spider = Spider(self.spider_name)
# DeltaFetch creates .db files named after the spider's name
self.temp_dir = tempfile.gettempdir()
self.db_path = os.path.join(self.temp_dir, '%s.db' % self.spider.name)
crawler = get_crawler(Spider)
self.stats = StatsCollector(crawler)
def test_core_stats_default_stats_collector(self, mock_datetime):
fixed_datetime = datetime(2019, 12, 1, 11, 38)
mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
self.crawler.stats = StatsCollector(self.crawler)
ext = CoreStats.from_crawler(self.crawler)
ext.spider_opened(self.spider)
ext.item_scraped({}, self.spider)
ext.response_received(self.spider)
ext.item_dropped({}, self.spider, ZeroDivisionError())
ext.spider_closed(self.spider, 'finished')
self.assertEqual(
ext.stats._stats, {
'start_time': fixed_datetime,
'finish_time': fixed_datetime,
'item_scraped_count': 1,
'response_received_count': 1,
'item_dropped_count': 1,
'item_dropped_reasons_count/ZeroDivisionError': 1,
'finish_reason': 'finished',
'elapsed_time_seconds': 0.0,
})
def setUp(self):
self.spider = Spider('df_tests')
self.temp_dir = tempfile.gettempdir()
self.db_path = os.path.join(self.temp_dir, 'df_tests.db')
crawler = get_crawler(Spider)
self.stats = StatsCollector(crawler)
